2024-11-16 05:53:55,631 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-16 05:53:55,646 main DEBUG Took 0.013130 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 05:53:55,647 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 05:53:55,647 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 05:53:55,648 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 05:53:55,650 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,658 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 05:53:55,672 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,673 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,673 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,674 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,675 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,675 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,676 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,677 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,677 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,678 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,678 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,678 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,679 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,679 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,679 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,680 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,680 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,681 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,681 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,681 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,681 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:53:55,682 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,682 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 05:53:55,683 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:53:55,685 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 05:53:55,686 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 05:53:55,687 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 05:53:55,688 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 05:53:55,688 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 05:53:55,696 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 05:53:55,699 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 05:53:55,701 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 05:53:55,702 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 05:53:55,702 main DEBUG createAppenders(={Console}) 2024-11-16 05:53:55,703 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-16 05:53:55,704 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-16 05:53:55,704 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-16 05:53:55,705 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 05:53:55,705 main DEBUG OutputStream closed 2024-11-16 05:53:55,706 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 05:53:55,706 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 05:53:55,706 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-16 05:53:55,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 05:53:55,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 05:53:55,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 05:53:55,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 05:53:55,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 05:53:55,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 05:53:55,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 05:53:55,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 05:53:55,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 05:53:55,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 05:53:55,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 05:53:55,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 05:53:55,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 05:53:55,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 05:53:55,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 05:53:55,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 05:53:55,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 05:53:55,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 05:53:55,792 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 05:53:55,792 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-16 05:53:55,792 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 05:53:55,793 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-16T05:53:55,805 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-16 05:53:55,807 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 05:53:55,808 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T05:53:56,043 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522 2024-11-16T05:53:56,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T05:53:56,079 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/cluster_41502c83-d9ee-d043-a50f-6b7260140340, deleteOnExit=true 2024-11-16T05:53:56,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T05:53:56,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/test.cache.data in system properties and HBase conf 2024-11-16T05:53:56,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T05:53:56,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/hadoop.log.dir in system properties and HBase conf 2024-11-16T05:53:56,083 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T05:53:56,083 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T05:53:56,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T05:53:56,177 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T05:53:56,264 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:53:56,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T05:53:56,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T05:53:56,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T05:53:56,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T05:53:56,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T05:53:56,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T05:53:56,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T05:53:56,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T05:53:56,271 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T05:53:56,271 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/nfs.dump.dir in system properties and HBase conf 2024-11-16T05:53:56,272 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/java.io.tmpdir in system properties and HBase conf 2024-11-16T05:53:56,272 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T05:53:56,272 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T05:53:56,273 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T05:53:56,988 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T05:53:57,066 INFO [Time-limited test {}] log.Log(170): Logging initialized @2124ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T05:53:57,136 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:53:57,201 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:53:57,224 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:53:57,225 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:53:57,227 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T05:53:57,240 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:53:57,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:53:57,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:53:57,412 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/java.io.tmpdir/jetty-localhost-35805-hadoop-hdfs-3_4_1-tests_jar-_-any-16734654872128408317/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T05:53:57,418 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:35805} 2024-11-16T05:53:57,419 INFO [Time-limited test {}] server.Server(415): Started @2478ms 2024-11-16T05:53:57,757 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:53:57,765 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:53:57,767 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:53:57,767 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:53:57,767 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T05:53:57,768 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:53:57,769 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:53:57,868 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/java.io.tmpdir/jetty-localhost-33651-hadoop-hdfs-3_4_1-tests_jar-_-any-17890598137750982578/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:53:57,869 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:33651} 2024-11-16T05:53:57,869 INFO [Time-limited test {}] server.Server(415): Started @2929ms 2024-11-16T05:53:57,917 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T05:53:58,022 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:53:58,029 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:53:58,035 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:53:58,035 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:53:58,035 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T05:53:58,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:53:58,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:53:58,138 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/java.io.tmpdir/jetty-localhost-35657-hadoop-hdfs-3_4_1-tests_jar-_-any-4462928497458408708/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:53:58,139 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:35657} 2024-11-16T05:53:58,139 INFO [Time-limited test {}] server.Server(415): Started @3199ms 2024-11-16T05:53:58,142 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T05:53:58,186 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:53:58,191 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:53:58,194 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:53:58,195 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:53:58,195 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T05:53:58,197 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:53:58,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:53:58,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/java.io.tmpdir/jetty-localhost-33349-hadoop-hdfs-3_4_1-tests_jar-_-any-275216648616940487/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:53:58,313 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:33349} 2024-11-16T05:53:58,313 INFO [Time-limited test {}] server.Server(415): Started @3373ms 2024-11-16T05:53:58,314 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/cluster_41502c83-d9ee-d043-a50f-6b7260140340/data/data1/current/BP-889549478-172.17.0.2-1731736436787/current, will proceed with Du for space computation calculation, 2024-11-16T05:53:58,314 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/cluster_41502c83-d9ee-d043-a50f-6b7260140340/data/data3/current/BP-889549478-172.17.0.2-1731736436787/current, will proceed with Du for space computation calculation, 2024-11-16T05:53:58,314 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/cluster_41502c83-d9ee-d043-a50f-6b7260140340/data/data2/current/BP-889549478-172.17.0.2-1731736436787/current, will proceed with Du for space computation calculation, 2024-11-16T05:53:58,314 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/cluster_41502c83-d9ee-d043-a50f-6b7260140340/data/data4/current/BP-889549478-172.17.0.2-1731736436787/current, will proceed with Du for space computation calculation, 2024-11-16T05:53:58,316 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T05:53:58,352 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T05:53:58,352 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T05:53:58,415 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb78d68f51130d626 with lease ID 0x155ed7c71a21a573: Processing first storage report for DS-9304333b-0d3c-4c8a-9a98-d919495de0c9 from datanode DatanodeRegistration(127.0.0.1:34749, datanodeUuid=360f772f-6ddc-494e-9138-26f2efdb78e1, infoPort=40917, infoSecurePort=0, ipcPort=40757, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787) 2024-11-16T05:53:58,417 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb78d68f51130d626 with lease ID 0x155ed7c71a21a573: from storage DS-9304333b-0d3c-4c8a-9a98-d919495de0c9 node DatanodeRegistration(127.0.0.1:34749, datanodeUuid=360f772f-6ddc-494e-9138-26f2efdb78e1, infoPort=40917, infoSecurePort=0, ipcPort=40757, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T05:53:58,418 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x39beddff20155dc1 with lease ID 0x155ed7c71a21a574: Processing first storage report for DS-76cee6b1-1fc1-4942-af99-4fd1252edb11 from datanode DatanodeRegistration(127.0.0.1:46509, datanodeUuid=b5cf3804-6e2a-4b76-abfb-f1f84fd4b1c2, infoPort=33953, infoSecurePort=0, ipcPort=43153, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787) 2024-11-16T05:53:58,418 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x39beddff20155dc1 with lease ID 0x155ed7c71a21a574: from storage DS-76cee6b1-1fc1-4942-af99-4fd1252edb11 node DatanodeRegistration(127.0.0.1:46509, datanodeUuid=b5cf3804-6e2a-4b76-abfb-f1f84fd4b1c2, infoPort=33953, infoSecurePort=0, ipcPort=43153, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T05:53:58,418 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb78d68f51130d626 with lease ID 0x155ed7c71a21a573: Processing first storage report for DS-cac6cbb8-42f9-4de8-bdef-416740609e3b from datanode DatanodeRegistration(127.0.0.1:34749, datanodeUuid=360f772f-6ddc-494e-9138-26f2efdb78e1, infoPort=40917, infoSecurePort=0, ipcPort=40757, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787) 2024-11-16T05:53:58,419 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb78d68f51130d626 with lease ID 0x155ed7c71a21a573: from storage DS-cac6cbb8-42f9-4de8-bdef-416740609e3b node DatanodeRegistration(127.0.0.1:34749, datanodeUuid=360f772f-6ddc-494e-9138-26f2efdb78e1, infoPort=40917, infoSecurePort=0, ipcPort=40757, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T05:53:58,419 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x39beddff20155dc1 with lease ID 0x155ed7c71a21a574: Processing first storage report for DS-22d2cd38-9873-4e20-b2e5-8c9532604d45 from datanode DatanodeRegistration(127.0.0.1:46509, datanodeUuid=b5cf3804-6e2a-4b76-abfb-f1f84fd4b1c2, infoPort=33953, infoSecurePort=0, ipcPort=43153, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787) 2024-11-16T05:53:58,419 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x39beddff20155dc1 with lease ID 0x155ed7c71a21a574: from storage DS-22d2cd38-9873-4e20-b2e5-8c9532604d45 node DatanodeRegistration(127.0.0.1:46509, datanodeUuid=b5cf3804-6e2a-4b76-abfb-f1f84fd4b1c2, infoPort=33953, infoSecurePort=0, ipcPort=43153, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T05:53:58,433 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/cluster_41502c83-d9ee-d043-a50f-6b7260140340/data/data5/current/BP-889549478-172.17.0.2-1731736436787/current, will proceed with Du for space computation calculation, 2024-11-16T05:53:58,434 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/cluster_41502c83-d9ee-d043-a50f-6b7260140340/data/data6/current/BP-889549478-172.17.0.2-1731736436787/current, will proceed with Du for space computation calculation, 2024-11-16T05:53:58,462 WARN [Thread-121 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T05:53:58,467 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9001b941845b9027 with lease ID 0x155ed7c71a21a575: Processing first storage report for DS-977bdc24-b9f7-4fd5-8b69-f87ba6a67efb from datanode DatanodeRegistration(127.0.0.1:39925, datanodeUuid=2dbab2b9-24be-467b-a0ac-355595687497, infoPort=42883, infoSecurePort=0, ipcPort=33541, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787) 2024-11-16T05:53:58,467 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9001b941845b9027 with lease ID 0x155ed7c71a21a575: from storage DS-977bdc24-b9f7-4fd5-8b69-f87ba6a67efb node DatanodeRegistration(127.0.0.1:39925, datanodeUuid=2dbab2b9-24be-467b-a0ac-355595687497, infoPort=42883, infoSecurePort=0, ipcPort=33541, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T05:53:58,468 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9001b941845b9027 with lease ID 0x155ed7c71a21a575: Processing first storage report for DS-3750d306-81fe-4c2d-8cdc-89272d75e0a0 from datanode DatanodeRegistration(127.0.0.1:39925, datanodeUuid=2dbab2b9-24be-467b-a0ac-355595687497, infoPort=42883, infoSecurePort=0, ipcPort=33541, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787) 2024-11-16T05:53:58,468 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9001b941845b9027 with lease ID 0x155ed7c71a21a575: from storage DS-3750d306-81fe-4c2d-8cdc-89272d75e0a0 node DatanodeRegistration(127.0.0.1:39925, datanodeUuid=2dbab2b9-24be-467b-a0ac-355595687497, infoPort=42883, infoSecurePort=0, ipcPort=33541, storageInfo=lv=-57;cid=testClusterID;nsid=637226361;c=1731736436787), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T05:53:58,655 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522 2024-11-16T05:53:58,724 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/cluster_41502c83-d9ee-d043-a50f-6b7260140340/zookeeper_0, clientPort=57132, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/cluster_41502c83-d9ee-d043-a50f-6b7260140340/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f3162240-b678-ff4f-633a-47ed05628522/cluster_41502c83-d9ee-d043-a50f-6b7260140340/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T05:53:58,734 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57132 2024-11-16T05:53:58,747 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:53:58,750 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:53:58,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741825_1001 (size=7) 2024-11-16T05:53:58,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741825_1001 (size=7) 2024-11-16T05:53:58,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741825_1001 (size=7) 2024-11-16T05:53:59,381 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 with version=8 2024-11-16T05:53:59,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/hbase-staging 2024-11-16T05:53:59,456 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T05:53:59,651 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:53:59,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:53:59,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:53:59,663 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:53:59,664 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:53:59,664 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:53:59,783 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T05:53:59,835 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T05:53:59,843 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T05:53:59,847 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:53:59,871 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 85947 (auto-detected) 2024-11-16T05:53:59,872 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T05:53:59,889 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40501 2024-11-16T05:53:59,909 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40501 connecting to ZooKeeper ensemble=127.0.0.1:57132 2024-11-16T05:53:59,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405010x0, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:53:59,938 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40501-0x100471864220000 connected 2024-11-16T05:53:59,961 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:53:59,967 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:53:59,983 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:53:59,986 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79, hbase.cluster.distributed=false 2024-11-16T05:54:00,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:54:00,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40501 2024-11-16T05:54:00,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40501 2024-11-16T05:54:00,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40501 2024-11-16T05:54:00,017 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40501 2024-11-16T05:54:00,017 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40501 2024-11-16T05:54:00,111 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:54:00,113 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:00,113 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:00,113 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:54:00,113 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:00,113 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:54:00,116 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:54:00,118 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:54:00,119 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36299 2024-11-16T05:54:00,121 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36299 connecting to ZooKeeper ensemble=127.0.0.1:57132 2024-11-16T05:54:00,122 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:54:00,124 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:54:00,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:362990x0, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:54:00,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:362990x0, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:54:00,129 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36299-0x100471864220001 connected 2024-11-16T05:54:00,132 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:54:00,139 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:54:00,141 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:54:00,145 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:54:00,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36299 2024-11-16T05:54:00,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36299 2024-11-16T05:54:00,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36299 2024-11-16T05:54:00,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36299 2024-11-16T05:54:00,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36299 2024-11-16T05:54:00,166 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:54:00,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:00,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:00,167 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:54:00,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:00,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:54:00,168 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:54:00,168 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:54:00,169 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42285 2024-11-16T05:54:00,171 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42285 connecting to ZooKeeper ensemble=127.0.0.1:57132 2024-11-16T05:54:00,173 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:54:00,176 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:54:00,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422850x0, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:54:00,184 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42285-0x100471864220002 connected 2024-11-16T05:54:00,184 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:54:00,184 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:54:00,185 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:54:00,186 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:54:00,188 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:54:00,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42285 2024-11-16T05:54:00,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42285 2024-11-16T05:54:00,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42285 2024-11-16T05:54:00,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42285 2024-11-16T05:54:00,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42285 2024-11-16T05:54:00,211 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:54:00,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:00,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:00,212 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:54:00,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:00,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:54:00,213 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:54:00,213 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:54:00,215 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37837 2024-11-16T05:54:00,218 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37837 connecting to ZooKeeper ensemble=127.0.0.1:57132 2024-11-16T05:54:00,219 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:54:00,222 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:54:00,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378370x0, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:54:00,229 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:378370x0, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:54:00,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37837-0x100471864220003 connected 2024-11-16T05:54:00,229 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:54:00,230 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:54:00,231 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:54:00,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:54:00,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37837 2024-11-16T05:54:00,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37837 2024-11-16T05:54:00,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37837 2024-11-16T05:54:00,235 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37837 2024-11-16T05:54:00,236 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37837 2024-11-16T05:54:00,249 DEBUG [M:0;3456ee6a3164:40501 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3456ee6a3164:40501 2024-11-16T05:54:00,250 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3456ee6a3164,40501,1731736439501 2024-11-16T05:54:00,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:54:00,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:54:00,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:54:00,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:54:00,259 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3456ee6a3164,40501,1731736439501 2024-11-16T05:54:00,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:54:00,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:54:00,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:54:00,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,281 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T05:54:00,282 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3456ee6a3164,40501,1731736439501 from backup master directory 2024-11-16T05:54:00,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:54:00,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:54:00,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3456ee6a3164,40501,1731736439501 2024-11-16T05:54:00,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:54:00,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:54:00,285 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:54:00,286 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3456ee6a3164,40501,1731736439501 2024-11-16T05:54:00,287 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T05:54:00,288 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T05:54:00,339 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/hbase.id] with ID: 5e221f76-f3d5-4494-a674-1fee78496748 2024-11-16T05:54:00,339 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/.tmp/hbase.id 2024-11-16T05:54:00,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741826_1002 (size=42) 2024-11-16T05:54:00,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741826_1002 (size=42) 2024-11-16T05:54:00,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741826_1002 (size=42) 2024-11-16T05:54:00,356 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/.tmp/hbase.id]:[hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/hbase.id] 2024-11-16T05:54:00,397 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:54:00,402 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T05:54:00,419 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-16T05:54:00,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741827_1003 (size=196) 2024-11-16T05:54:00,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741827_1003 (size=196) 2024-11-16T05:54:00,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741827_1003 (size=196) 2024-11-16T05:54:00,459 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:54:00,461 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T05:54:00,474 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:00,478 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:54:00,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741828_1004 (size=1189) 2024-11-16T05:54:00,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741828_1004 (size=1189) 2024-11-16T05:54:00,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741828_1004 (size=1189) 2024-11-16T05:54:00,525 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/data/master/store 2024-11-16T05:54:00,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741829_1005 (size=34) 2024-11-16T05:54:00,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741829_1005 (size=34) 2024-11-16T05:54:00,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741829_1005 (size=34) 2024-11-16T05:54:00,552 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T05:54:00,556 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:00,557 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:54:00,557 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:00,558 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:00,560 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:54:00,560 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:00,560 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:00,561 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731736440557Disabling compacts and flushes for region at 1731736440557Disabling writes for close at 1731736440560 (+3 ms)Writing region close event to WAL at 1731736440560Closed at 1731736440560 2024-11-16T05:54:00,564 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/data/master/store/.initializing 2024-11-16T05:54:00,564 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/WALs/3456ee6a3164,40501,1731736439501 2024-11-16T05:54:00,573 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:54:00,590 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C40501%2C1731736439501, suffix=, logDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/WALs/3456ee6a3164,40501,1731736439501, archiveDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/oldWALs, maxLogs=10 2024-11-16T05:54:00,613 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/WALs/3456ee6a3164,40501,1731736439501/3456ee6a3164%2C40501%2C1731736439501.1731736440596, exclude list is [], retry=0 2024-11-16T05:54:00,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34749,DS-9304333b-0d3c-4c8a-9a98-d919495de0c9,DISK] 2024-11-16T05:54:00,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39925,DS-977bdc24-b9f7-4fd5-8b69-f87ba6a67efb,DISK] 2024-11-16T05:54:00,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46509,DS-76cee6b1-1fc1-4942-af99-4fd1252edb11,DISK] 2024-11-16T05:54:00,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T05:54:00,670 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/WALs/3456ee6a3164,40501,1731736439501/3456ee6a3164%2C40501%2C1731736439501.1731736440596 2024-11-16T05:54:00,671 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40917:40917),(127.0.0.1/127.0.0.1:33953:33953),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-16T05:54:00,672 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:54:00,672 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:00,675 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,676 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,736 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T05:54:00,739 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:00,742 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:00,742 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T05:54:00,745 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:00,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:00,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T05:54:00,749 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:00,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:00,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T05:54:00,753 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:00,754 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:00,755 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,758 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,760 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,765 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,766 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,770 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:54:00,774 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:54:00,779 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:00,780 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60719770, jitterRate=-0.0952049195766449}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:54:00,792 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731736440687Initializing all the Stores at 1731736440690 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736440690Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736440691 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736440691Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736440691Cleaning up temporary data from old regions at 1731736440766 (+75 ms)Region opened successfully at 1731736440792 (+26 ms) 2024-11-16T05:54:00,794 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T05:54:00,838 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b05ae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:54:00,868 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T05:54:00,878 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:00,879 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:00,881 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:00,883 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T05:54:00,887 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T05:54:00,887 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T05:54:00,912 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T05:54:00,921 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T05:54:00,923 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T05:54:00,926 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T05:54:00,928 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T05:54:00,929 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T05:54:00,932 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T05:54:00,935 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T05:54:00,938 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T05:54:00,939 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T05:54:00,941 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T05:54:00,958 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T05:54:00,960 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T05:54:00,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:54:00,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:54:00,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:54:00,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:54:00,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,968 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3456ee6a3164,40501,1731736439501, sessionid=0x100471864220000, setting cluster-up flag (Was=false) 2024-11-16T05:54:00,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,986 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T05:54:00,989 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,40501,1731736439501 2024-11-16T05:54:00,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:00,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:01,001 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T05:54:01,003 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,40501,1731736439501 2024-11-16T05:54:01,009 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T05:54:01,039 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(746): ClusterId : 5e221f76-f3d5-4494-a674-1fee78496748 2024-11-16T05:54:01,039 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(746): ClusterId : 5e221f76-f3d5-4494-a674-1fee78496748 2024-11-16T05:54:01,039 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(746): ClusterId : 5e221f76-f3d5-4494-a674-1fee78496748 2024-11-16T05:54:01,041 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:54:01,041 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:54:01,041 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:54:01,047 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:54:01,047 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:54:01,047 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:54:01,048 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:54:01,048 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:54:01,048 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:54:01,053 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:54:01,053 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:54:01,053 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:54:01,054 DEBUG [RS:0;3456ee6a3164:36299 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8bdffd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:54:01,054 DEBUG [RS:2;3456ee6a3164:37837 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@778fe8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:54:01,054 DEBUG [RS:1;3456ee6a3164:42285 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3647a81e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:54:01,071 DEBUG [RS:0;3456ee6a3164:36299 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3456ee6a3164:36299 2024-11-16T05:54:01,071 DEBUG [RS:1;3456ee6a3164:42285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;3456ee6a3164:42285 2024-11-16T05:54:01,071 DEBUG [RS:2;3456ee6a3164:37837 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;3456ee6a3164:37837 2024-11-16T05:54:01,075 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:54:01,075 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:54:01,075 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:54:01,075 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:54:01,075 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:54:01,075 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:54:01,075 DEBUG [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:54:01,075 DEBUG [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:54:01,075 DEBUG [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:54:01,078 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,40501,1731736439501 with port=36299, startcode=1731736440080 2024-11-16T05:54:01,078 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,40501,1731736439501 with port=37837, startcode=1731736440211 2024-11-16T05:54:01,078 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,40501,1731736439501 with port=42285, startcode=1731736440166 2024-11-16T05:54:01,083 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:01,088 DEBUG [RS:0;3456ee6a3164:36299 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:54:01,088 DEBUG [RS:2;3456ee6a3164:37837 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:54:01,088 DEBUG [RS:1;3456ee6a3164:42285 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:54:01,092 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T05:54:01,099 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T05:54:01,105 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3456ee6a3164,40501,1731736439501 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T05:54:01,114 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:54:01,115 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:54:01,115 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:54:01,115 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:54:01,115 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3456ee6a3164:0, corePoolSize=10, maxPoolSize=10 2024-11-16T05:54:01,116 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,116 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:54:01,116 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,117 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736471117 2024-11-16T05:54:01,120 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T05:54:01,121 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32789, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:54:01,121 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35555, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:54:01,121 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54371, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:54:01,121 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T05:54:01,123 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:01,124 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T05:54:01,125 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T05:54:01,126 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T05:54:01,126 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T05:54:01,126 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T05:54:01,128 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:01,129 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:01,128 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,130 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T05:54:01,131 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T05:54:01,132 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T05:54:01,132 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T05:54:01,135 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:01,135 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T05:54:01,136 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T05:54:01,136 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:01,141 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736441137,5,FailOnTimeoutGroup] 2024-11-16T05:54:01,142 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736441141,5,FailOnTimeoutGroup] 2024-11-16T05:54:01,142 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,142 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T05:54:01,143 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,144 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741831_1007 (size=1321) 2024-11-16T05:54:01,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741831_1007 (size=1321) 2024-11-16T05:54:01,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741831_1007 (size=1321) 2024-11-16T05:54:01,202 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T05:54:01,202 DEBUG [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T05:54:01,202 DEBUG [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T05:54:01,202 DEBUG [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T05:54:01,202 WARN [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T05:54:01,202 WARN [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T05:54:01,202 WARN [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T05:54:01,203 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:01,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741832_1008 (size=32) 2024-11-16T05:54:01,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741832_1008 (size=32) 2024-11-16T05:54:01,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741832_1008 (size=32) 2024-11-16T05:54:01,217 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:01,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T05:54:01,221 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:54:01,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:01,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:01,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T05:54:01,225 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:54:01,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:01,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:01,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T05:54:01,229 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:54:01,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:01,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:01,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T05:54:01,233 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:54:01,233 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:01,234 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:01,234 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:54:01,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740 2024-11-16T05:54:01,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740 2024-11-16T05:54:01,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:54:01,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:54:01,241 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:54:01,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:54:01,247 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:01,249 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65070306, jitterRate=-0.03037688136100769}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:54:01,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731736441217Initializing all the Stores at 1731736441218 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736441219 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736441219Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736441219Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736441219Cleaning up temporary data from old regions at 1731736441240 (+21 ms)Region opened successfully at 1731736441254 (+14 ms) 2024-11-16T05:54:01,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:54:01,254 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:54:01,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:54:01,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:54:01,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:54:01,256 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:54:01,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736441254Disabling compacts and flushes for region at 1731736441254Disabling writes for close at 1731736441254Writing region close event to WAL at 1731736441255 (+1 ms)Closed at 1731736441256 (+1 ms) 2024-11-16T05:54:01,260 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:01,260 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T05:54:01,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T05:54:01,280 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:01,283 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T05:54:01,304 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,40501,1731736439501 with port=36299, startcode=1731736440080 2024-11-16T05:54:01,304 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,40501,1731736439501 with port=37837, startcode=1731736440211 2024-11-16T05:54:01,304 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,40501,1731736439501 with port=42285, startcode=1731736440166 2024-11-16T05:54:01,306 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,37837,1731736440211 2024-11-16T05:54:01,308 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:01,316 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,42285,1731736440166 2024-11-16T05:54:01,316 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:01,317 DEBUG [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:01,317 DEBUG [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43797 2024-11-16T05:54:01,317 DEBUG [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:54:01,319 DEBUG [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:01,319 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,36299,1731736440080 2024-11-16T05:54:01,319 DEBUG [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43797 2024-11-16T05:54:01,319 DEBUG [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:54:01,320 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:01,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:54:01,324 DEBUG [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:01,324 DEBUG [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43797 2024-11-16T05:54:01,325 DEBUG [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:54:01,327 DEBUG [RS:2;3456ee6a3164:37837 {}] zookeeper.ZKUtil(111): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,37837,1731736440211 2024-11-16T05:54:01,327 WARN [RS:2;3456ee6a3164:37837 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:54:01,327 INFO [RS:2;3456ee6a3164:37837 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:54:01,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:54:01,327 DEBUG [RS:1;3456ee6a3164:42285 {}] zookeeper.ZKUtil(111): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,42285,1731736440166 2024-11-16T05:54:01,327 WARN [RS:1;3456ee6a3164:42285 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:54:01,327 DEBUG [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,37837,1731736440211 2024-11-16T05:54:01,327 INFO [RS:1;3456ee6a3164:42285 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:54:01,328 DEBUG [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,42285,1731736440166 2024-11-16T05:54:01,328 DEBUG [RS:0;3456ee6a3164:36299 {}] zookeeper.ZKUtil(111): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,36299,1731736440080 2024-11-16T05:54:01,328 WARN [RS:0;3456ee6a3164:36299 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:54:01,328 INFO [RS:0;3456ee6a3164:36299 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:54:01,328 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,37837,1731736440211] 2024-11-16T05:54:01,328 DEBUG [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080 2024-11-16T05:54:01,328 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,42285,1731736440166] 2024-11-16T05:54:01,329 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,36299,1731736440080] 2024-11-16T05:54:01,351 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:54:01,351 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:54:01,351 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:54:01,368 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:54:01,368 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:54:01,368 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:54:01,374 INFO [RS:1;3456ee6a3164:42285 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:54:01,374 INFO [RS:2;3456ee6a3164:37837 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:54:01,374 INFO [RS:0;3456ee6a3164:36299 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:54:01,374 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,374 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,374 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,375 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:54:01,376 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:54:01,376 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:54:01,381 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:54:01,381 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:54:01,381 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:54:01,383 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,383 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,383 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,383 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,383 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:54:01,384 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:54:01,384 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:54:01,384 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:54:01,384 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,384 DEBUG [RS:1;3456ee6a3164:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:54:01,385 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,385 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,385 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,385 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:54:01,385 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:54:01,385 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:54:01,385 DEBUG [RS:0;3456ee6a3164:36299 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:54:01,385 DEBUG [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:54:01,386 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,386 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,386 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,386 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,386 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,386 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,387 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,387 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,42285,1731736440166-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:54:01,387 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,387 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,387 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,387 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,36299,1731736440080-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:54:01,387 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,387 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,387 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,387 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,388 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,388 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,37837,1731736440211-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:54:01,406 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:54:01,408 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,37837,1731736440211-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,408 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,408 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.Replication(171): 3456ee6a3164,37837,1731736440211 started 2024-11-16T05:54:01,410 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:54:01,410 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:54:01,410 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,42285,1731736440166-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,410 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,36299,1731736440080-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,411 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,411 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.Replication(171): 3456ee6a3164,36299,1731736440080 started 2024-11-16T05:54:01,411 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,411 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.Replication(171): 3456ee6a3164,42285,1731736440166 started 2024-11-16T05:54:01,425 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,425 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,37837,1731736440211, RpcServer on 3456ee6a3164/172.17.0.2:37837, sessionid=0x100471864220003 2024-11-16T05:54:01,426 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:54:01,426 DEBUG [RS:2;3456ee6a3164:37837 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,37837,1731736440211 2024-11-16T05:54:01,426 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,37837,1731736440211' 2024-11-16T05:54:01,426 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:54:01,427 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:54:01,428 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:54:01,428 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:54:01,428 DEBUG [RS:2;3456ee6a3164:37837 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,37837,1731736440211 2024-11-16T05:54:01,428 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,37837,1731736440211' 2024-11-16T05:54:01,428 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:54:01,429 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:54:01,430 DEBUG [RS:2;3456ee6a3164:37837 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:54:01,430 INFO [RS:2;3456ee6a3164:37837 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:54:01,430 INFO [RS:2;3456ee6a3164:37837 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:54:01,433 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,433 INFO [RS:0;3456ee6a3164:36299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:01,433 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,42285,1731736440166, RpcServer on 3456ee6a3164/172.17.0.2:42285, sessionid=0x100471864220002 2024-11-16T05:54:01,433 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,36299,1731736440080, RpcServer on 3456ee6a3164/172.17.0.2:36299, sessionid=0x100471864220001 2024-11-16T05:54:01,434 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:54:01,434 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:54:01,434 DEBUG [RS:0;3456ee6a3164:36299 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,36299,1731736440080 2024-11-16T05:54:01,434 DEBUG [RS:1;3456ee6a3164:42285 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,42285,1731736440166 2024-11-16T05:54:01,434 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,36299,1731736440080' 2024-11-16T05:54:01,434 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,42285,1731736440166' 2024-11-16T05:54:01,434 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:54:01,434 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:54:01,434 WARN [3456ee6a3164:40501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T05:54:01,435 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:54:01,435 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:54:01,435 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:54:01,435 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:54:01,435 DEBUG [RS:1;3456ee6a3164:42285 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,42285,1731736440166 2024-11-16T05:54:01,435 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,42285,1731736440166' 2024-11-16T05:54:01,435 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:54:01,435 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:54:01,435 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:54:01,435 DEBUG [RS:0;3456ee6a3164:36299 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,36299,1731736440080 2024-11-16T05:54:01,436 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,36299,1731736440080' 2024-11-16T05:54:01,436 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:54:01,436 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:54:01,436 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:54:01,436 DEBUG [RS:1;3456ee6a3164:42285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:54:01,436 INFO [RS:1;3456ee6a3164:42285 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:54:01,436 INFO [RS:1;3456ee6a3164:42285 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:54:01,436 DEBUG [RS:0;3456ee6a3164:36299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:54:01,436 INFO [RS:0;3456ee6a3164:36299 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:54:01,437 INFO [RS:0;3456ee6a3164:36299 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:54:01,534 INFO [RS:2;3456ee6a3164:37837 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:54:01,537 INFO [RS:2;3456ee6a3164:37837 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C37837%2C1731736440211, suffix=, logDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,37837,1731736440211, archiveDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/oldWALs, maxLogs=32 2024-11-16T05:54:01,537 INFO [RS:1;3456ee6a3164:42285 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:54:01,537 INFO [RS:0;3456ee6a3164:36299 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:54:01,540 INFO [RS:0;3456ee6a3164:36299 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C36299%2C1731736440080, suffix=, logDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080, archiveDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/oldWALs, maxLogs=32 2024-11-16T05:54:01,540 INFO [RS:1;3456ee6a3164:42285 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C42285%2C1731736440166, suffix=, logDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,42285,1731736440166, archiveDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/oldWALs, maxLogs=32 2024-11-16T05:54:01,553 DEBUG [RS:2;3456ee6a3164:37837 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,37837,1731736440211/3456ee6a3164%2C37837%2C1731736440211.1731736441540, exclude list is [], retry=0 2024-11-16T05:54:01,555 DEBUG [RS:1;3456ee6a3164:42285 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,42285,1731736440166/3456ee6a3164%2C42285%2C1731736440166.1731736441542, exclude list is [], retry=0 2024-11-16T05:54:01,558 DEBUG [RS:0;3456ee6a3164:36299 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080/3456ee6a3164%2C36299%2C1731736440080.1731736441542, exclude list is [], retry=0 2024-11-16T05:54:01,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39925,DS-977bdc24-b9f7-4fd5-8b69-f87ba6a67efb,DISK] 2024-11-16T05:54:01,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46509,DS-76cee6b1-1fc1-4942-af99-4fd1252edb11,DISK] 2024-11-16T05:54:01,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34749,DS-9304333b-0d3c-4c8a-9a98-d919495de0c9,DISK] 2024-11-16T05:54:01,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34749,DS-9304333b-0d3c-4c8a-9a98-d919495de0c9,DISK] 2024-11-16T05:54:01,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39925,DS-977bdc24-b9f7-4fd5-8b69-f87ba6a67efb,DISK] 2024-11-16T05:54:01,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46509,DS-76cee6b1-1fc1-4942-af99-4fd1252edb11,DISK] 2024-11-16T05:54:01,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46509,DS-76cee6b1-1fc1-4942-af99-4fd1252edb11,DISK] 2024-11-16T05:54:01,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39925,DS-977bdc24-b9f7-4fd5-8b69-f87ba6a67efb,DISK] 2024-11-16T05:54:01,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34749,DS-9304333b-0d3c-4c8a-9a98-d919495de0c9,DISK] 2024-11-16T05:54:01,571 INFO [RS:1;3456ee6a3164:42285 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,42285,1731736440166/3456ee6a3164%2C42285%2C1731736440166.1731736441542 2024-11-16T05:54:01,571 INFO [RS:2;3456ee6a3164:37837 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,37837,1731736440211/3456ee6a3164%2C37837%2C1731736440211.1731736441540 2024-11-16T05:54:01,572 INFO [RS:0;3456ee6a3164:36299 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080/3456ee6a3164%2C36299%2C1731736440080.1731736441542 2024-11-16T05:54:01,572 DEBUG [RS:1;3456ee6a3164:42285 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33953:33953),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:40917:40917)] 2024-11-16T05:54:01,572 DEBUG [RS:2;3456ee6a3164:37837 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33953:33953),(127.0.0.1/127.0.0.1:40917:40917),(127.0.0.1/127.0.0.1:42883:42883)] 2024-11-16T05:54:01,572 DEBUG [RS:0;3456ee6a3164:36299 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33953:33953),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:40917:40917)] 2024-11-16T05:54:01,690 DEBUG [3456ee6a3164:40501 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-16T05:54:01,700 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:54:01,705 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:54:01,705 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:54:01,706 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T05:54:01,706 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:54:01,706 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:54:01,706 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T05:54:01,706 INFO [3456ee6a3164:40501 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:54:01,706 INFO [3456ee6a3164:40501 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:54:01,706 INFO [3456ee6a3164:40501 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T05:54:01,706 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:54:01,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:01,718 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,36299,1731736440080, state=OPENING 2024-11-16T05:54:01,723 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T05:54:01,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:01,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:01,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:01,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:01,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:01,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:01,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:01,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:01,728 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:01,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:01,910 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:54:01,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45833, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:54:01,924 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T05:54:01,924 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:54:01,925 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T05:54:01,927 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C36299%2C1731736440080.meta, suffix=.meta, logDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080, archiveDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/oldWALs, maxLogs=32 2024-11-16T05:54:01,943 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, exclude list is [], retry=0 2024-11-16T05:54:01,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46509,DS-76cee6b1-1fc1-4942-af99-4fd1252edb11,DISK] 2024-11-16T05:54:01,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39925,DS-977bdc24-b9f7-4fd5-8b69-f87ba6a67efb,DISK] 2024-11-16T05:54:01,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34749,DS-9304333b-0d3c-4c8a-9a98-d919495de0c9,DISK] 2024-11-16T05:54:01,950 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta 2024-11-16T05:54:01,950 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33953:33953),(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:40917:40917)] 2024-11-16T05:54:01,951 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:54:01,952 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T05:54:01,954 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T05:54:01,959 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T05:54:01,962 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T05:54:01,962 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:01,963 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T05:54:01,963 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T05:54:01,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T05:54:01,967 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:54:01,967 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:01,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:01,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T05:54:01,970 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:54:01,970 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:01,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:01,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T05:54:01,973 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:54:01,973 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:01,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:01,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T05:54:01,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:54:01,975 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:01,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:01,977 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:54:01,978 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740 2024-11-16T05:54:01,981 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740 2024-11-16T05:54:01,984 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:54:01,984 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:54:01,985 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:54:01,987 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:54:01,989 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72162960, jitterRate=0.07531189918518066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:54:01,989 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T05:54:01,990 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731736441963Writing region info on filesystem at 1731736441963Initializing all the Stores at 1731736441965 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736441965Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736441965Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736441965Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736441965Cleaning up temporary data from old regions at 1731736441984 (+19 ms)Running coprocessor post-open hooks at 1731736441989 (+5 ms)Region opened successfully at 1731736441990 (+1 ms) 2024-11-16T05:54:01,997 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731736441899 2024-11-16T05:54:02,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:02,010 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T05:54:02,010 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T05:54:02,013 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,36299,1731736440080, state=OPEN 2024-11-16T05:54:02,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:02,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:02,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:02,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:02,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:02,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:02,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:02,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:02,018 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:02,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T05:54:02,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,36299,1731736440080 in 290 msec 2024-11-16T05:54:02,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T05:54:02,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 760 msec 2024-11-16T05:54:02,037 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:02,037 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T05:54:02,060 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:02,061 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1] 2024-11-16T05:54:02,081 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:54:02,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44149, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:54:02,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0640 sec 2024-11-16T05:54:02,103 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731736442103, completionTime=-1 2024-11-16T05:54:02,106 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-16T05:54:02,107 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:02,137 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-16T05:54:02,137 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736502137 2024-11-16T05:54:02,138 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736562138 2024-11-16T05:54:02,138 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-11-16T05:54:02,139 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:02,147 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40501,1731736439501-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:02,147 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40501,1731736439501-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:02,147 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40501,1731736439501-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:02,149 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3456ee6a3164:40501, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:02,150 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:02,150 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:02,156 DEBUG [master/3456ee6a3164:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T05:54:02,180 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.894sec 2024-11-16T05:54:02,181 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T05:54:02,183 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T05:54:02,183 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T05:54:02,184 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T05:54:02,184 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T05:54:02,185 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40501,1731736439501-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:54:02,185 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40501,1731736439501-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T05:54:02,189 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T05:54:02,190 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T05:54:02,191 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40501,1731736439501-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:54:02,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117ba4ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:54:02,252 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T05:54:02,252 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T05:54:02,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,40501,-1 for getting cluster id 2024-11-16T05:54:02,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:54:02,266 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5e221f76-f3d5-4494-a674-1fee78496748' 2024-11-16T05:54:02,268 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:54:02,269 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5e221f76-f3d5-4494-a674-1fee78496748" 2024-11-16T05:54:02,269 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4728fb40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:54:02,269 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,40501,-1] 2024-11-16T05:54:02,272 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:54:02,274 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:02,275 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:54:02,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc9d8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:54:02,278 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:02,286 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1] 2024-11-16T05:54:02,286 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:54:02,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:54:02,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3456ee6a3164,40501,1731736439501 2024-11-16T05:54:02,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:54:02,339 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,40501,1731736439501 2024-11-16T05:54:02,341 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f1054bd 2024-11-16T05:54:02,343 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:54:02,347 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:54:02,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:54:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:02,370 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:54:02,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-16T05:54:02,376 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:02,379 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:54:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:54:02,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741837_1013 (size=358) 2024-11-16T05:54:02,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741837_1013 (size=358) 2024-11-16T05:54:02,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741837_1013 (size=358) 2024-11-16T05:54:02,412 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1309086cc4268bb14438bcc29f5b63ff, NAME => 'test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,422 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fa4cd934bc5e4b765d7bb157c1c600c0, NAME => 'test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,423 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d63eb285e07e85739c793802fc7058c2, NAME => 'test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741838_1014 (size=42) 2024-11-16T05:54:02,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741838_1014 (size=42) 2024-11-16T05:54:02,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741838_1014 (size=42) 2024-11-16T05:54:02,437 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,437 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1309086cc4268bb14438bcc29f5b63ff, disabling compactions & flushes 2024-11-16T05:54:02,437 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:02,437 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:02,437 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. after waiting 0 ms 2024-11-16T05:54:02,437 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:02,437 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:02,437 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1309086cc4268bb14438bcc29f5b63ff: Waiting for close lock at 1731736442437Disabling compacts and flushes for region at 1731736442437Disabling writes for close at 1731736442437Writing region close event to WAL at 1731736442437Closed at 1731736442437 2024-11-16T05:54:02,438 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ba2670ce022e672b048093a692f3cb8, NAME => 'test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741839_1015 (size=45) 2024-11-16T05:54:02,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741839_1015 (size=45) 2024-11-16T05:54:02,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741839_1015 (size=45) 2024-11-16T05:54:02,457 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,457 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d63eb285e07e85739c793802fc7058c2, disabling compactions & flushes 2024-11-16T05:54:02,457 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:02,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:02,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. after waiting 0 ms 2024-11-16T05:54:02,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:02,458 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:02,458 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d63eb285e07e85739c793802fc7058c2: Waiting for close lock at 1731736442457Disabling compacts and flushes for region at 1731736442457Disabling writes for close at 1731736442458 (+1 ms)Writing region close event to WAL at 1731736442458Closed at 1731736442458 2024-11-16T05:54:02,459 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6a7892b4caedb942fa23cc8a2f9d3b15, NAME => 'test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741840_1016 (size=45) 2024-11-16T05:54:02,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741840_1016 (size=45) 2024-11-16T05:54:02,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741840_1016 (size=45) 2024-11-16T05:54:02,462 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,463 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fa4cd934bc5e4b765d7bb157c1c600c0, disabling compactions & flushes 2024-11-16T05:54:02,463 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:02,463 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:02,463 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. after waiting 0 ms 2024-11-16T05:54:02,463 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:02,463 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:02,463 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fa4cd934bc5e4b765d7bb157c1c600c0: Waiting for close lock at 1731736442463Disabling compacts and flushes for region at 1731736442463Disabling writes for close at 1731736442463Writing region close event to WAL at 1731736442463Closed at 1731736442463 2024-11-16T05:54:02,464 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ceb8a4dcf270b48605e90af8f3b8eb2c, NAME => 'test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741841_1017 (size=45) 2024-11-16T05:54:02,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741841_1017 (size=45) 2024-11-16T05:54:02,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741841_1017 (size=45) 2024-11-16T05:54:02,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,470 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6ba2670ce022e672b048093a692f3cb8, disabling compactions & flushes 2024-11-16T05:54:02,470 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:02,471 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:02,471 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. after waiting 0 ms 2024-11-16T05:54:02,471 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:02,471 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:02,471 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ba2670ce022e672b048093a692f3cb8: Waiting for close lock at 1731736442470Disabling compacts and flushes for region at 1731736442470Disabling writes for close at 1731736442471 (+1 ms)Writing region close event to WAL at 1731736442471Closed at 1731736442471 2024-11-16T05:54:02,472 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 83571bc94223ec9eaf85f80b241d7135, NAME => 'test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741843_1019 (size=45) 2024-11-16T05:54:02,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741843_1019 (size=45) 2024-11-16T05:54:02,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741843_1019 (size=45) 2024-11-16T05:54:02,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741842_1018 (size=45) 2024-11-16T05:54:02,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741842_1018 (size=45) 2024-11-16T05:54:02,490 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6a7892b4caedb942fa23cc8a2f9d3b15, disabling compactions & flushes 2024-11-16T05:54:02,491 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15. 2024-11-16T05:54:02,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15. 2024-11-16T05:54:02,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15. after waiting 0 ms 2024-11-16T05:54:02,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15. 2024-11-16T05:54:02,491 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15. 2024-11-16T05:54:02,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6a7892b4caedb942fa23cc8a2f9d3b15: Waiting for close lock at 1731736442491Disabling compacts and flushes for region at 1731736442491Disabling writes for close at 1731736442491Writing region close event to WAL at 1731736442491Closed at 1731736442491 2024-11-16T05:54:02,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741842_1018 (size=45) 2024-11-16T05:54:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:54:02,492 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 3c48229c22c1b63769411754c30d2dfe, NAME => 'test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,492 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,492 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ceb8a4dcf270b48605e90af8f3b8eb2c, disabling compactions & flushes 2024-11-16T05:54:02,492 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:02,493 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:02,493 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. after waiting 0 ms 2024-11-16T05:54:02,493 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:02,493 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:02,493 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ceb8a4dcf270b48605e90af8f3b8eb2c: Waiting for close lock at 1731736442492Disabling compacts and flushes for region at 1731736442492Disabling writes for close at 1731736442493 (+1 ms)Writing region close event to WAL at 1731736442493Closed at 1731736442493 2024-11-16T05:54:02,494 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 20aedf13a99ac07f411f959c4c4bed3f, NAME => 'test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741844_1020 (size=45) 2024-11-16T05:54:02,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741844_1020 (size=45) 2024-11-16T05:54:02,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741844_1020 (size=45) 2024-11-16T05:54:02,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741845_1021 (size=45) 2024-11-16T05:54:02,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741845_1021 (size=45) 2024-11-16T05:54:02,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741846_1022 (size=45) 2024-11-16T05:54:02,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741846_1022 (size=45) 2024-11-16T05:54:02,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741845_1021 (size=45) 2024-11-16T05:54:02,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741846_1022 (size=45) 2024-11-16T05:54:02,516 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,516 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 20aedf13a99ac07f411f959c4c4bed3f, disabling compactions & flushes 2024-11-16T05:54:02,516 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:02,516 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:02,516 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. after waiting 0 ms 2024-11-16T05:54:02,516 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:02,516 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:02,516 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 20aedf13a99ac07f411f959c4c4bed3f: Waiting for close lock at 1731736442516Disabling compacts and flushes for region at 1731736442516Disabling writes for close at 1731736442516Writing region close event to WAL at 1731736442516Closed at 1731736442516 2024-11-16T05:54:02,517 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,517 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => da031f93bd094d06fcd97a63a4675032, NAME => 'test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,517 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 3c48229c22c1b63769411754c30d2dfe, disabling compactions & flushes 2024-11-16T05:54:02,517 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:02,517 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:02,517 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. after waiting 0 ms 2024-11-16T05:54:02,517 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:02,517 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:02,518 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 3c48229c22c1b63769411754c30d2dfe: Waiting for close lock at 1731736442517Disabling compacts and flushes for region at 1731736442517Disabling writes for close at 1731736442517Writing region close event to WAL at 1731736442517Closed at 1731736442517 2024-11-16T05:54:02,518 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f391d2492a2a9b1c649bcab4e64504c7, NAME => 'test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741847_1023 (size=45) 2024-11-16T05:54:02,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741847_1023 (size=45) 2024-11-16T05:54:02,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741848_1024 (size=45) 2024-11-16T05:54:02,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741848_1024 (size=45) 2024-11-16T05:54:02,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing da031f93bd094d06fcd97a63a4675032, disabling compactions & flushes 2024-11-16T05:54:02,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741847_1023 (size=45) 2024-11-16T05:54:02,538 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:02,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:02,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. after waiting 0 ms 2024-11-16T05:54:02,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:02,538 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:02,539 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for da031f93bd094d06fcd97a63a4675032: Waiting for close lock at 1731736442538Disabling compacts and flushes for region at 1731736442538Disabling writes for close at 1731736442538Writing region close event to WAL at 1731736442538Closed at 1731736442538 2024-11-16T05:54:02,539 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 83edddaeaac4b866615839a3e7d98099, NAME => 'test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741848_1024 (size=45) 2024-11-16T05:54:02,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741849_1025 (size=45) 2024-11-16T05:54:02,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741849_1025 (size=45) 2024-11-16T05:54:02,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741849_1025 (size=45) 2024-11-16T05:54:02,556 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,556 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 83edddaeaac4b866615839a3e7d98099, disabling compactions & flushes 2024-11-16T05:54:02,556 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:02,557 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:02,557 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. after waiting 0 ms 2024-11-16T05:54:02,557 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:02,557 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:02,557 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 83edddaeaac4b866615839a3e7d98099: Waiting for close lock at 1731736442556Disabling compacts and flushes for region at 1731736442556Disabling writes for close at 1731736442557 (+1 ms)Writing region close event to WAL at 1731736442557Closed at 1731736442557 2024-11-16T05:54:02,558 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => bf20fbe8271ec5dfab659b24954f2df5, NAME => 'test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741850_1026 (size=45) 2024-11-16T05:54:02,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741850_1026 (size=45) 2024-11-16T05:54:02,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741850_1026 (size=45) 2024-11-16T05:54:02,571 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,571 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing bf20fbe8271ec5dfab659b24954f2df5, disabling compactions & flushes 2024-11-16T05:54:02,571 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:02,571 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:02,571 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. after waiting 0 ms 2024-11-16T05:54:02,571 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:02,571 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:02,571 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for bf20fbe8271ec5dfab659b24954f2df5: Waiting for close lock at 1731736442571Disabling compacts and flushes for region at 1731736442571Disabling writes for close at 1731736442571Writing region close event to WAL at 1731736442571Closed at 1731736442571 2024-11-16T05:54:02,572 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fdf8bdb7617bd8bb875a900a673e25fc, NAME => 'test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741851_1027 (size=45) 2024-11-16T05:54:02,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741851_1027 (size=45) 2024-11-16T05:54:02,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741851_1027 (size=45) 2024-11-16T05:54:02,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fdf8bdb7617bd8bb875a900a673e25fc, disabling compactions & flushes 2024-11-16T05:54:02,585 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:02,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:02,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. after waiting 0 ms 2024-11-16T05:54:02,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:02,585 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:02,586 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fdf8bdb7617bd8bb875a900a673e25fc: Waiting for close lock at 1731736442585Disabling compacts and flushes for region at 1731736442585Disabling writes for close at 1731736442585Writing region close event to WAL at 1731736442585Closed at 1731736442585 2024-11-16T05:54:02,586 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ac4bc85feea9c2820d87cf4b219cb17c, NAME => 'test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741852_1028 (size=45) 2024-11-16T05:54:02,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741852_1028 (size=45) 2024-11-16T05:54:02,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741852_1028 (size=45) 2024-11-16T05:54:02,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ac4bc85feea9c2820d87cf4b219cb17c, disabling compactions & flushes 2024-11-16T05:54:02,597 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:02,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:02,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. after waiting 0 ms 2024-11-16T05:54:02,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:02,597 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:02,597 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ac4bc85feea9c2820d87cf4b219cb17c: Waiting for close lock at 1731736442597Disabling compacts and flushes for region at 1731736442597Disabling writes for close at 1731736442597Writing region close event to WAL at 1731736442597Closed at 1731736442597 2024-11-16T05:54:02,598 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 52d4c19e400f7867ccce41418f251b74, NAME => 'test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741853_1029 (size=45) 2024-11-16T05:54:02,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741853_1029 (size=45) 2024-11-16T05:54:02,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741853_1029 (size=45) 2024-11-16T05:54:02,608 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,608 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 52d4c19e400f7867ccce41418f251b74, disabling compactions & flushes 2024-11-16T05:54:02,609 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. 2024-11-16T05:54:02,609 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. 2024-11-16T05:54:02,609 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. after waiting 0 ms 2024-11-16T05:54:02,609 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. 2024-11-16T05:54:02,609 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. 2024-11-16T05:54:02,609 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 52d4c19e400f7867ccce41418f251b74: Waiting for close lock at 1731736442608Disabling compacts and flushes for region at 1731736442608Disabling writes for close at 1731736442609 (+1 ms)Writing region close event to WAL at 1731736442609Closed at 1731736442609 2024-11-16T05:54:02,609 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7df86faabb66872de7c73da2806f6ac3, NAME => 'test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741854_1030 (size=45) 2024-11-16T05:54:02,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741854_1030 (size=45) 2024-11-16T05:54:02,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741854_1030 (size=45) 2024-11-16T05:54:02,624 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7df86faabb66872de7c73da2806f6ac3, disabling compactions & flushes 2024-11-16T05:54:02,625 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:02,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:02,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. after waiting 0 ms 2024-11-16T05:54:02,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:02,625 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:02,625 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7df86faabb66872de7c73da2806f6ac3: Waiting for close lock at 1731736442625Disabling compacts and flushes for region at 1731736442625Disabling writes for close at 1731736442625Writing region close event to WAL at 1731736442625Closed at 1731736442625 2024-11-16T05:54:02,626 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e2536762964ceee99f3f786e0dad8403, NAME => 'test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741855_1031 (size=45) 2024-11-16T05:54:02,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741855_1031 (size=45) 2024-11-16T05:54:02,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741855_1031 (size=45) 2024-11-16T05:54:02,638 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,638 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e2536762964ceee99f3f786e0dad8403, disabling compactions & flushes 2024-11-16T05:54:02,638 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:02,638 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:02,638 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. after waiting 0 ms 2024-11-16T05:54:02,638 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:02,638 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:02,638 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e2536762964ceee99f3f786e0dad8403: Waiting for close lock at 1731736442638Disabling compacts and flushes for region at 1731736442638Disabling writes for close at 1731736442638Writing region close event to WAL at 1731736442638Closed at 1731736442638 2024-11-16T05:54:02,639 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a62176f809b89002f2152e96ba4bef24, NAME => 'test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741856_1032 (size=45) 2024-11-16T05:54:02,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741856_1032 (size=45) 2024-11-16T05:54:02,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741856_1032 (size=45) 2024-11-16T05:54:02,649 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,649 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a62176f809b89002f2152e96ba4bef24, disabling compactions & flushes 2024-11-16T05:54:02,650 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. 2024-11-16T05:54:02,650 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. 2024-11-16T05:54:02,650 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. after waiting 0 ms 2024-11-16T05:54:02,650 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. 2024-11-16T05:54:02,650 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. 2024-11-16T05:54:02,650 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a62176f809b89002f2152e96ba4bef24: Waiting for close lock at 1731736442649Disabling compacts and flushes for region at 1731736442649Disabling writes for close at 1731736442650 (+1 ms)Writing region close event to WAL at 1731736442650Closed at 1731736442650 2024-11-16T05:54:02,651 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f060fbfb898c418effc5c41d59bdb94f, NAME => 'test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741857_1033 (size=45) 2024-11-16T05:54:02,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741857_1033 (size=45) 2024-11-16T05:54:02,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741857_1033 (size=45) 2024-11-16T05:54:02,668 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,668 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f060fbfb898c418effc5c41d59bdb94f, disabling compactions & flushes 2024-11-16T05:54:02,668 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:02,668 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:02,668 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. after waiting 0 ms 2024-11-16T05:54:02,668 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:02,668 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:02,668 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f060fbfb898c418effc5c41d59bdb94f: Waiting for close lock at 1731736442668Disabling compacts and flushes for region at 1731736442668Disabling writes for close at 1731736442668Writing region close event to WAL at 1731736442668Closed at 1731736442668 2024-11-16T05:54:02,669 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c342bb403770b75ff0a582854ab71fbe, NAME => 'test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741858_1034 (size=45) 2024-11-16T05:54:02,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741858_1034 (size=45) 2024-11-16T05:54:02,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741858_1034 (size=45) 2024-11-16T05:54:02,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c342bb403770b75ff0a582854ab71fbe, disabling compactions & flushes 2024-11-16T05:54:02,680 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:02,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:02,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. after waiting 0 ms 2024-11-16T05:54:02,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:02,680 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:02,680 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c342bb403770b75ff0a582854ab71fbe: Waiting for close lock at 1731736442680Disabling compacts and flushes for region at 1731736442680Disabling writes for close at 1731736442680Writing region close event to WAL at 1731736442680Closed at 1731736442680 2024-11-16T05:54:02,681 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0760749bb2566c6d8e6b56f164a53ebe, NAME => 'test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741859_1035 (size=45) 2024-11-16T05:54:02,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741859_1035 (size=45) 2024-11-16T05:54:02,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741859_1035 (size=45) 2024-11-16T05:54:02,692 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,692 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0760749bb2566c6d8e6b56f164a53ebe, disabling compactions & flushes 2024-11-16T05:54:02,692 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe. 2024-11-16T05:54:02,692 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe. 2024-11-16T05:54:02,692 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe. after waiting 0 ms 2024-11-16T05:54:02,692 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe. 2024-11-16T05:54:02,692 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe. 2024-11-16T05:54:02,692 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0760749bb2566c6d8e6b56f164a53ebe: Waiting for close lock at 1731736442692Disabling compacts and flushes for region at 1731736442692Disabling writes for close at 1731736442692Writing region close event to WAL at 1731736442692Closed at 1731736442692 2024-11-16T05:54:02,693 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8e3c5f7060b7491b30cd6cf5617a89c2, NAME => 'test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741860_1036 (size=45) 2024-11-16T05:54:02,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741860_1036 (size=45) 2024-11-16T05:54:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:54:02,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741860_1036 (size=45) 2024-11-16T05:54:02,705 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,705 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8e3c5f7060b7491b30cd6cf5617a89c2, disabling compactions & flushes 2024-11-16T05:54:02,706 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:02,706 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:02,706 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. after waiting 0 ms 2024-11-16T05:54:02,706 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:02,706 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:02,706 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8e3c5f7060b7491b30cd6cf5617a89c2: Waiting for close lock at 1731736442705Disabling compacts and flushes for region at 1731736442705Disabling writes for close at 1731736442706 (+1 ms)Writing region close event to WAL at 1731736442706Closed at 1731736442706 2024-11-16T05:54:02,706 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 67d7f5593e349615c92abcb96869f4ba, NAME => 'test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741861_1037 (size=45) 2024-11-16T05:54:02,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741861_1037 (size=45) 2024-11-16T05:54:02,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741861_1037 (size=45) 2024-11-16T05:54:02,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 67d7f5593e349615c92abcb96869f4ba, disabling compactions & flushes 2024-11-16T05:54:02,718 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:02,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:02,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. after waiting 0 ms 2024-11-16T05:54:02,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:02,718 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:02,718 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 67d7f5593e349615c92abcb96869f4ba: Waiting for close lock at 1731736442718Disabling compacts and flushes for region at 1731736442718Disabling writes for close at 1731736442718Writing region close event to WAL at 1731736442718Closed at 1731736442718 2024-11-16T05:54:02,719 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => df6f5bd741b507073cb45265746143fa, NAME => 'test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741862_1038 (size=45) 2024-11-16T05:54:02,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741862_1038 (size=45) 2024-11-16T05:54:02,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741862_1038 (size=45) 2024-11-16T05:54:02,731 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,731 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing df6f5bd741b507073cb45265746143fa, disabling compactions & flushes 2024-11-16T05:54:02,731 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:02,731 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:02,731 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. after waiting 0 ms 2024-11-16T05:54:02,731 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:02,731 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:02,731 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for df6f5bd741b507073cb45265746143fa: Waiting for close lock at 1731736442731Disabling compacts and flushes for region at 1731736442731Disabling writes for close at 1731736442731Writing region close event to WAL at 1731736442731Closed at 1731736442731 2024-11-16T05:54:02,732 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d426c5a1ddcc32067a58955e792d6dae, NAME => 'test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79 2024-11-16T05:54:02,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741863_1039 (size=42) 2024-11-16T05:54:02,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741863_1039 (size=42) 2024-11-16T05:54:02,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741863_1039 (size=42) 2024-11-16T05:54:02,742 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,742 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d426c5a1ddcc32067a58955e792d6dae, disabling compactions & flushes 2024-11-16T05:54:02,743 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:02,743 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:02,743 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. after waiting 0 ms 2024-11-16T05:54:02,743 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:02,743 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:02,743 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d426c5a1ddcc32067a58955e792d6dae: Waiting for close lock at 1731736442742Disabling compacts and flushes for region at 1731736442742Disabling writes for close at 1731736442743 (+1 ms)Writing region close event to WAL at 1731736442743Closed at 1731736442743 2024-11-16T05:54:02,904 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,904 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 83571bc94223ec9eaf85f80b241d7135, disabling compactions & flushes 2024-11-16T05:54:02,904 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:02,904 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:02,904 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. after waiting 0 ms 2024-11-16T05:54:02,904 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:02,904 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:02,905 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 83571bc94223ec9eaf85f80b241d7135: Waiting for close lock at 1731736442904Disabling compacts and flushes for region at 1731736442904Disabling writes for close at 1731736442904Writing region close event to WAL at 1731736442904Closed at 1731736442904 2024-11-16T05:54:02,940 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:02,941 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f391d2492a2a9b1c649bcab4e64504c7, disabling compactions & flushes 2024-11-16T05:54:02,941 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7. 2024-11-16T05:54:02,941 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7. 2024-11-16T05:54:02,941 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7. after waiting 0 ms 2024-11-16T05:54:02,941 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7. 2024-11-16T05:54:02,941 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7. 2024-11-16T05:54:02,942 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f391d2492a2a9b1c649bcab4e64504c7: Waiting for close lock at 1731736442941Disabling compacts and flushes for region at 1731736442941Disabling writes for close at 1731736442941Writing region close event to WAL at 1731736442941Closed at 1731736442941 2024-11-16T05:54:02,949 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:54:02,955 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,955 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,956 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,956 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,956 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,956 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,958 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,958 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,958 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,958 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,958 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,959 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,959 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,959 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,959 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,960 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,960 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,960 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,960 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,960 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:02,960 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731736442949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736442949"}]},"ts":"1731736442949"} 2024-11-16T05:54:03,002 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-16T05:54:03,004 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:54:03,006 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736443004"}]},"ts":"1731736443004"} 2024-11-16T05:54:03,011 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-16T05:54:03,012 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:54:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:54:03,028 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:54:03,028 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:54:03,028 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T05:54:03,028 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:54:03,028 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:54:03,028 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T05:54:03,028 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:54:03,028 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:54:03,028 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T05:54:03,028 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:54:03,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1309086cc4268bb14438bcc29f5b63ff, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa4cd934bc5e4b765d7bb157c1c600c0, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6ba2670ce022e672b048093a692f3cb8, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ceb8a4dcf270b48605e90af8f3b8eb2c, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=83571bc94223ec9eaf85f80b241d7135, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=20aedf13a99ac07f411f959c4c4bed3f, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=da031f93bd094d06fcd97a63a4675032, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=83edddaeaac4b866615839a3e7d98099, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fdf8bdb7617bd8bb875a900a673e25fc, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ac4bc85feea9c2820d87cf4b219cb17c, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7df86faabb66872de7c73da2806f6ac3, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e2536762964ceee99f3f786e0dad8403, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f060fbfb898c418effc5c41d59bdb94f, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c342bb403770b75ff0a582854ab71fbe, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e3c5f7060b7491b30cd6cf5617a89c2, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=67d7f5593e349615c92abcb96869f4ba, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d426c5a1ddcc32067a58955e792d6dae, ASSIGN}] 2024-11-16T05:54:03,044 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d426c5a1ddcc32067a58955e792d6dae, ASSIGN 2024-11-16T05:54:03,045 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d426c5a1ddcc32067a58955e792d6dae, ASSIGN; state=OFFLINE, location=3456ee6a3164,37837,1731736440211; forceNewPlan=false, retain=false 2024-11-16T05:54:03,047 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:03,048 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN; state=OFFLINE, location=3456ee6a3164,36299,1731736440080; forceNewPlan=false, retain=false 2024-11-16T05:54:03,050 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=67d7f5593e349615c92abcb96869f4ba, ASSIGN 2024-11-16T05:54:03,052 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=67d7f5593e349615c92abcb96869f4ba, ASSIGN; state=OFFLINE, location=3456ee6a3164,42285,1731736440166; forceNewPlan=false, retain=false 2024-11-16T05:54:03,053 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e3c5f7060b7491b30cd6cf5617a89c2, ASSIGN 2024-11-16T05:54:03,055 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8e3c5f7060b7491b30cd6cf5617a89c2, ASSIGN; state=OFFLINE, location=3456ee6a3164,37837,1731736440211; forceNewPlan=false, retain=false 2024-11-16T05:54:03,057 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:03,058 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN; state=OFFLINE, location=3456ee6a3164,36299,1731736440080; forceNewPlan=false, retain=false 2024-11-16T05:54:03,060 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c342bb403770b75ff0a582854ab71fbe, ASSIGN 2024-11-16T05:54:03,062 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c342bb403770b75ff0a582854ab71fbe, ASSIGN; state=OFFLINE, location=3456ee6a3164,42285,1731736440166; forceNewPlan=false, retain=false 2024-11-16T05:54:03,064 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f060fbfb898c418effc5c41d59bdb94f, ASSIGN 2024-11-16T05:54:03,066 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f060fbfb898c418effc5c41d59bdb94f, ASSIGN; state=OFFLINE, location=3456ee6a3164,37837,1731736440211; forceNewPlan=false, retain=false 2024-11-16T05:54:03,067 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:03,069 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN; state=OFFLINE, location=3456ee6a3164,36299,1731736440080; forceNewPlan=false, retain=false 2024-11-16T05:54:03,071 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e2536762964ceee99f3f786e0dad8403, ASSIGN 2024-11-16T05:54:03,072 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e2536762964ceee99f3f786e0dad8403, ASSIGN; state=OFFLINE, location=3456ee6a3164,42285,1731736440166; forceNewPlan=false, retain=false 2024-11-16T05:54:03,075 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7df86faabb66872de7c73da2806f6ac3, ASSIGN 2024-11-16T05:54:03,076 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7df86faabb66872de7c73da2806f6ac3, ASSIGN; state=OFFLINE, location=3456ee6a3164,37837,1731736440211; forceNewPlan=false, retain=false 2024-11-16T05:54:03,078 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:03,079 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN; state=OFFLINE, location=3456ee6a3164,36299,1731736440080; forceNewPlan=false, retain=false 2024-11-16T05:54:03,081 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ac4bc85feea9c2820d87cf4b219cb17c, ASSIGN 2024-11-16T05:54:03,082 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ac4bc85feea9c2820d87cf4b219cb17c, ASSIGN; state=OFFLINE, location=3456ee6a3164,42285,1731736440166; forceNewPlan=false, retain=false 2024-11-16T05:54:03,083 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fdf8bdb7617bd8bb875a900a673e25fc, ASSIGN 2024-11-16T05:54:03,085 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fdf8bdb7617bd8bb875a900a673e25fc, ASSIGN; state=OFFLINE, location=3456ee6a3164,37837,1731736440211; forceNewPlan=false, retain=false 2024-11-16T05:54:03,086 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:03,087 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN; state=OFFLINE, location=3456ee6a3164,36299,1731736440080; forceNewPlan=false, retain=false 2024-11-16T05:54:03,089 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=83edddaeaac4b866615839a3e7d98099, ASSIGN 2024-11-16T05:54:03,090 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=83edddaeaac4b866615839a3e7d98099, ASSIGN; state=OFFLINE, location=3456ee6a3164,42285,1731736440166; forceNewPlan=false, retain=false 2024-11-16T05:54:03,092 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:03,094 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN; state=OFFLINE, location=3456ee6a3164,36299,1731736440080; forceNewPlan=false, retain=false 2024-11-16T05:54:03,095 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=da031f93bd094d06fcd97a63a4675032, ASSIGN 2024-11-16T05:54:03,096 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=da031f93bd094d06fcd97a63a4675032, ASSIGN; state=OFFLINE, location=3456ee6a3164,37837,1731736440211; forceNewPlan=false, retain=false 2024-11-16T05:54:03,098 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=20aedf13a99ac07f411f959c4c4bed3f, ASSIGN 2024-11-16T05:54:03,100 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=20aedf13a99ac07f411f959c4c4bed3f, ASSIGN; state=OFFLINE, location=3456ee6a3164,42285,1731736440166; forceNewPlan=false, retain=false 2024-11-16T05:54:03,101 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:03,103 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN; state=OFFLINE, location=3456ee6a3164,36299,1731736440080; forceNewPlan=false, retain=false 2024-11-16T05:54:03,105 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=83571bc94223ec9eaf85f80b241d7135, ASSIGN 2024-11-16T05:54:03,106 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=83571bc94223ec9eaf85f80b241d7135, ASSIGN; state=OFFLINE, location=3456ee6a3164,42285,1731736440166; forceNewPlan=false, retain=false 2024-11-16T05:54:03,108 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ceb8a4dcf270b48605e90af8f3b8eb2c, ASSIGN 2024-11-16T05:54:03,109 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ceb8a4dcf270b48605e90af8f3b8eb2c, ASSIGN; state=OFFLINE, location=3456ee6a3164,37837,1731736440211; forceNewPlan=false, retain=false 2024-11-16T05:54:03,111 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:03,112 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN; state=OFFLINE, location=3456ee6a3164,36299,1731736440080; forceNewPlan=false, retain=false 2024-11-16T05:54:03,114 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6ba2670ce022e672b048093a692f3cb8, ASSIGN 2024-11-16T05:54:03,116 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6ba2670ce022e672b048093a692f3cb8, ASSIGN; state=OFFLINE, location=3456ee6a3164,42285,1731736440166; forceNewPlan=false, retain=false 2024-11-16T05:54:03,118 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, ASSIGN 2024-11-16T05:54:03,120 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, ASSIGN; state=OFFLINE, location=3456ee6a3164,36299,1731736440080; forceNewPlan=false, retain=false 2024-11-16T05:54:03,122 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa4cd934bc5e4b765d7bb157c1c600c0, ASSIGN 2024-11-16T05:54:03,124 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fa4cd934bc5e4b765d7bb157c1c600c0, ASSIGN; state=OFFLINE, location=3456ee6a3164,37837,1731736440211; forceNewPlan=false, retain=false 2024-11-16T05:54:03,126 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1309086cc4268bb14438bcc29f5b63ff, ASSIGN 2024-11-16T05:54:03,128 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1309086cc4268bb14438bcc29f5b63ff, ASSIGN; state=OFFLINE, location=3456ee6a3164,42285,1731736440166; forceNewPlan=false, retain=false 2024-11-16T05:54:03,199 INFO [3456ee6a3164:40501 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-16T05:54:03,200 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=67d7f5593e349615c92abcb96869f4ba, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,202 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c342bb403770b75ff0a582854ab71fbe, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,202 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e2536762964ceee99f3f786e0dad8403, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,204 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=83edddaeaac4b866615839a3e7d98099, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=67d7f5593e349615c92abcb96869f4ba, ASSIGN because future has completed 2024-11-16T05:54:03,205 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67d7f5593e349615c92abcb96869f4ba, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:03,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c342bb403770b75ff0a582854ab71fbe, ASSIGN because future has completed 2024-11-16T05:54:03,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e2536762964ceee99f3f786e0dad8403, ASSIGN because future has completed 2024-11-16T05:54:03,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ac4bc85feea9c2820d87cf4b219cb17c, ASSIGN because future has completed 2024-11-16T05:54:03,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=83edddaeaac4b866615839a3e7d98099, ASSIGN because future has completed 2024-11-16T05:54:03,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83edddaeaac4b866615839a3e7d98099, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:03,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=20aedf13a99ac07f411f959c4c4bed3f, ASSIGN because future has completed 2024-11-16T05:54:03,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20aedf13a99ac07f411f959c4c4bed3f, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:03,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac4bc85feea9c2820d87cf4b219cb17c, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:03,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2536762964ceee99f3f786e0dad8403, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:03,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure c342bb403770b75ff0a582854ab71fbe, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:03,242 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=83571bc94223ec9eaf85f80b241d7135, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=6ba2670ce022e672b048093a692f3cb8, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1309086cc4268bb14438bcc29f5b63ff, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=d426c5a1ddcc32067a58955e792d6dae, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=f060fbfb898c418effc5c41d59bdb94f, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,246 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=7df86faabb66872de7c73da2806f6ac3, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=83571bc94223ec9eaf85f80b241d7135, ASSIGN because future has completed 2024-11-16T05:54:03,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83571bc94223ec9eaf85f80b241d7135, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:03,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6ba2670ce022e672b048093a692f3cb8, ASSIGN because future has completed 2024-11-16T05:54:03,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1309086cc4268bb14438bcc29f5b63ff, ASSIGN because future has completed 2024-11-16T05:54:03,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d426c5a1ddcc32067a58955e792d6dae, ASSIGN because future has completed 2024-11-16T05:54:03,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure d426c5a1ddcc32067a58955e792d6dae, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:03,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8e3c5f7060b7491b30cd6cf5617a89c2, ASSIGN because future has completed 2024-11-16T05:54:03,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f060fbfb898c418effc5c41d59bdb94f, ASSIGN because future has completed 2024-11-16T05:54:03,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7df86faabb66872de7c73da2806f6ac3, ASSIGN because future has completed 2024-11-16T05:54:03,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fdf8bdb7617bd8bb875a900a673e25fc, ASSIGN because future has completed 2024-11-16T05:54:03,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdf8bdb7617bd8bb875a900a673e25fc, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:03,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7df86faabb66872de7c73da2806f6ac3, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:03,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure f060fbfb898c418effc5c41d59bdb94f, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:03,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e3c5f7060b7491b30cd6cf5617a89c2, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:03,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1309086cc4268bb14438bcc29f5b63ff, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:03,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ba2670ce022e672b048093a692f3cb8, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:03,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=da031f93bd094d06fcd97a63a4675032, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,291 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=df6f5bd741b507073cb45265746143fa, regionState=OPENING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,294 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=da031f93bd094d06fcd97a63a4675032, ASSIGN because future has completed 2024-11-16T05:54:03,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure da031f93bd094d06fcd97a63a4675032, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:03,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ceb8a4dcf270b48605e90af8f3b8eb2c, ASSIGN because future has completed 2024-11-16T05:54:03,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fa4cd934bc5e4b765d7bb157c1c600c0, ASSIGN because future has completed 2024-11-16T05:54:03,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN because future has completed 2024-11-16T05:54:03,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN because future has completed 2024-11-16T05:54:03,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:03,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN because future has completed 2024-11-16T05:54:03,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN because future has completed 2024-11-16T05:54:03,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN because future has completed 2024-11-16T05:54:03,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:03,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52d4c19e400f7867ccce41418f251b74, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:03,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:03,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:03,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4cd934bc5e4b765d7bb157c1c600c0, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:03,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceb8a4dcf270b48605e90af8f3b8eb2c, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:03,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,340 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,341 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d63eb285e07e85739c793802fc7058c2, regionState=OPENING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN because future has completed 2024-11-16T05:54:03,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:03,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN because future has completed 2024-11-16T05:54:03,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN because future has completed 2024-11-16T05:54:03,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, ASSIGN because future has completed 2024-11-16T05:54:03,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d63eb285e07e85739c793802fc7058c2, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:03,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:03,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:03,361 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:54:03,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48477, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:54:03,368 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:03,369 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 83edddaeaac4b866615839a3e7d98099, NAME => 'test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-16T05:54:03,369 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,369 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,369 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,369 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,372 INFO [StoreOpener-83edddaeaac4b866615839a3e7d98099-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,374 INFO [StoreOpener-83edddaeaac4b866615839a3e7d98099-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83edddaeaac4b866615839a3e7d98099 columnFamilyName family 2024-11-16T05:54:03,374 DEBUG [StoreOpener-83edddaeaac4b866615839a3e7d98099-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,375 INFO [StoreOpener-83edddaeaac4b866615839a3e7d98099-1 {}] regionserver.HStore(327): Store=83edddaeaac4b866615839a3e7d98099/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,375 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,376 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,377 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,377 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,377 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,380 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,383 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/83edddaeaac4b866615839a3e7d98099/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,383 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 83edddaeaac4b866615839a3e7d98099; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70197500, jitterRate=0.04602426290512085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,384 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:03,385 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 83edddaeaac4b866615839a3e7d98099: Running coprocessor pre-open hook at 1731736443370Writing region info on filesystem at 1731736443370Initializing all the Stores at 1731736443372 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443372Cleaning up temporary data from old regions at 1731736443377 (+5 ms)Running coprocessor post-open hooks at 1731736443384 (+7 ms)Region opened successfully at 1731736443385 (+1 ms) 2024-11-16T05:54:03,386 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099., pid=32, masterSystemTime=1731736443360 2024-11-16T05:54:03,389 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:03,389 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:03,389 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:03,389 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => ac4bc85feea9c2820d87cf4b219cb17c, NAME => 'test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-16T05:54:03,389 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,390 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,390 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,390 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,390 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,393 INFO [StoreOpener-ac4bc85feea9c2820d87cf4b219cb17c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83edddaeaac4b866615839a3e7d98099, server=3456ee6a3164,42285,1731736440166 because future has completed 2024-11-16T05:54:03,395 INFO [StoreOpener-ac4bc85feea9c2820d87cf4b219cb17c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac4bc85feea9c2820d87cf4b219cb17c columnFamilyName family 2024-11-16T05:54:03,395 DEBUG [StoreOpener-ac4bc85feea9c2820d87cf4b219cb17c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,397 INFO [StoreOpener-ac4bc85feea9c2820d87cf4b219cb17c-1 {}] regionserver.HStore(327): Store=ac4bc85feea9c2820d87cf4b219cb17c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,397 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,399 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=16 2024-11-16T05:54:03,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 83edddaeaac4b866615839a3e7d98099, server=3456ee6a3164,42285,1731736440166 in 182 msec 2024-11-16T05:54:03,400 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,401 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,401 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=83edddaeaac4b866615839a3e7d98099, ASSIGN in 367 msec 2024-11-16T05:54:03,403 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,406 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/ac4bc85feea9c2820d87cf4b219cb17c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,407 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened ac4bc85feea9c2820d87cf4b219cb17c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71594167, jitterRate=0.06683622300624847}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,407 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:03,407 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for ac4bc85feea9c2820d87cf4b219cb17c: Running coprocessor pre-open hook at 1731736443390Writing region info on filesystem at 1731736443390Initializing all the Stores at 1731736443392 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443392Cleaning up temporary data from old regions at 1731736443401 (+9 ms)Running coprocessor post-open hooks at 1731736443407 (+6 ms)Region opened successfully at 1731736443407 2024-11-16T05:54:03,408 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:54:03,408 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c., pid=34, masterSystemTime=1731736443360 2024-11-16T05:54:03,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55913, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:54:03,411 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:03,411 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:03,411 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:03,412 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 1309086cc4268bb14438bcc29f5b63ff, NAME => 'test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-16T05:54:03,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,412 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,412 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,412 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,412 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,415 INFO [StoreOpener-1309086cc4268bb14438bcc29f5b63ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,415 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:03,415 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => fdf8bdb7617bd8bb875a900a673e25fc, NAME => 'test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-16T05:54:03,416 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,416 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,416 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,416 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac4bc85feea9c2820d87cf4b219cb17c, server=3456ee6a3164,42285,1731736440166 because future has completed 2024-11-16T05:54:03,418 INFO [StoreOpener-1309086cc4268bb14438bcc29f5b63ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1309086cc4268bb14438bcc29f5b63ff columnFamilyName family 2024-11-16T05:54:03,418 DEBUG [StoreOpener-1309086cc4268bb14438bcc29f5b63ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,418 INFO [StoreOpener-fdf8bdb7617bd8bb875a900a673e25fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,419 INFO [StoreOpener-1309086cc4268bb14438bcc29f5b63ff-1 {}] regionserver.HStore(327): Store=1309086cc4268bb14438bcc29f5b63ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,419 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,421 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,421 INFO [StoreOpener-fdf8bdb7617bd8bb875a900a673e25fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdf8bdb7617bd8bb875a900a673e25fc columnFamilyName family 2024-11-16T05:54:03,421 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,422 DEBUG [StoreOpener-fdf8bdb7617bd8bb875a900a673e25fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,422 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,422 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,422 INFO [StoreOpener-fdf8bdb7617bd8bb875a900a673e25fc-1 {}] regionserver.HStore(327): Store=fdf8bdb7617bd8bb875a900a673e25fc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=19 2024-11-16T05:54:03,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure ac4bc85feea9c2820d87cf4b219cb17c, server=3456ee6a3164,42285,1731736440166 in 193 msec 2024-11-16T05:54:03,425 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,425 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ac4bc85feea9c2820d87cf4b219cb17c, ASSIGN in 391 msec 2024-11-16T05:54:03,426 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,427 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,428 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,428 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,428 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/1309086cc4268bb14438bcc29f5b63ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,429 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 1309086cc4268bb14438bcc29f5b63ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73436635, jitterRate=0.09429113566875458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,429 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:03,430 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 1309086cc4268bb14438bcc29f5b63ff: Running coprocessor pre-open hook at 1731736443413Writing region info on filesystem at 1731736443413Initializing all the Stores at 1731736443414 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443414Cleaning up temporary data from old regions at 1731736443422 (+8 ms)Running coprocessor post-open hooks at 1731736443429 (+7 ms)Region opened successfully at 1731736443430 (+1 ms) 2024-11-16T05:54:03,430 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,431 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff., pid=43, masterSystemTime=1731736443360 2024-11-16T05:54:03,434 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/fdf8bdb7617bd8bb875a900a673e25fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,434 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:03,435 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:03,435 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:03,435 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 67d7f5593e349615c92abcb96869f4ba, NAME => 'test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-16T05:54:03,435 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened fdf8bdb7617bd8bb875a900a673e25fc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58993888, jitterRate=-0.12092256546020508}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,435 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:03,435 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,436 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,436 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for fdf8bdb7617bd8bb875a900a673e25fc: Running coprocessor pre-open hook at 1731736443416Writing region info on filesystem at 1731736443416Initializing all the Stores at 1731736443417 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443418 (+1 ms)Cleaning up temporary data from old regions at 1731736443428 (+10 ms)Running coprocessor post-open hooks at 1731736443435 (+7 ms)Region opened successfully at 1731736443436 (+1 ms) 2024-11-16T05:54:03,436 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,436 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,436 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,437 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc., pid=39, masterSystemTime=1731736443408 2024-11-16T05:54:03,439 INFO [StoreOpener-67d7f5593e349615c92abcb96869f4ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1309086cc4268bb14438bcc29f5b63ff, server=3456ee6a3164,42285,1731736440166 because future has completed 2024-11-16T05:54:03,441 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:03,441 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:03,441 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:03,441 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 7df86faabb66872de7c73da2806f6ac3, NAME => 'test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-16T05:54:03,442 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,442 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,442 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,442 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,443 INFO [StoreOpener-67d7f5593e349615c92abcb96869f4ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67d7f5593e349615c92abcb96869f4ba columnFamilyName family 2024-11-16T05:54:03,443 DEBUG [StoreOpener-67d7f5593e349615c92abcb96869f4ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,444 INFO [StoreOpener-67d7f5593e349615c92abcb96869f4ba-1 {}] regionserver.HStore(327): Store=67d7f5593e349615c92abcb96869f4ba/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,444 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=5 2024-11-16T05:54:03,445 INFO [StoreOpener-7df86faabb66872de7c73da2806f6ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1309086cc4268bb14438bcc29f5b63ff, server=3456ee6a3164,42285,1731736440166 in 160 msec 2024-11-16T05:54:03,447 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,447 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,448 INFO [StoreOpener-7df86faabb66872de7c73da2806f6ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7df86faabb66872de7c73da2806f6ac3 columnFamilyName family 2024-11-16T05:54:03,448 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,448 DEBUG [StoreOpener-7df86faabb66872de7c73da2806f6ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,448 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1309086cc4268bb14438bcc29f5b63ff, ASSIGN in 414 msec 2024-11-16T05:54:03,449 INFO [StoreOpener-7df86faabb66872de7c73da2806f6ac3-1 {}] regionserver.HStore(327): Store=7df86faabb66872de7c73da2806f6ac3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,450 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,451 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,451 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,452 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,453 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,453 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure fdf8bdb7617bd8bb875a900a673e25fc, server=3456ee6a3164,37837,1731736440211 because future has completed 2024-11-16T05:54:03,456 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/67d7f5593e349615c92abcb96869f4ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,457 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,458 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 67d7f5593e349615c92abcb96869f4ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66686973, jitterRate=-0.0062866657972335815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,458 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:03,459 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 67d7f5593e349615c92abcb96869f4ba: Running coprocessor pre-open hook at 1731736443436Writing region info on filesystem at 1731736443436Initializing all the Stores at 1731736443438 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443438Cleaning up temporary data from old regions at 1731736443448 (+10 ms)Running coprocessor post-open hooks at 1731736443459 (+11 ms)Region opened successfully at 1731736443459 2024-11-16T05:54:03,460 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba., pid=31, masterSystemTime=1731736443360 2024-11-16T05:54:03,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=18 2024-11-16T05:54:03,464 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/7df86faabb66872de7c73da2806f6ac3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,464 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:03,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure fdf8bdb7617bd8bb875a900a673e25fc, server=3456ee6a3164,37837,1731736440211 in 197 msec 2024-11-16T05:54:03,464 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:03,464 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:03,464 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ba2670ce022e672b048093a692f3cb8, NAME => 'test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-16T05:54:03,465 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,465 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,465 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,465 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 7df86faabb66872de7c73da2806f6ac3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75471492, jitterRate=0.12461286783218384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,465 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,465 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:03,465 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 7df86faabb66872de7c73da2806f6ac3: Running coprocessor pre-open hook at 1731736443442Writing region info on filesystem at 1731736443442Initializing all the Stores at 1731736443444 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443444Cleaning up temporary data from old regions at 1731736443453 (+9 ms)Running coprocessor post-open hooks at 1731736443465 (+12 ms)Region opened successfully at 1731736443465 2024-11-16T05:54:03,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fdf8bdb7617bd8bb875a900a673e25fc, ASSIGN in 432 msec 2024-11-16T05:54:03,467 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3., pid=40, masterSystemTime=1731736443408 2024-11-16T05:54:03,470 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,470 INFO [StoreOpener-6ba2670ce022e672b048093a692f3cb8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,470 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:03,471 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:03,471 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:03,471 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => fa4cd934bc5e4b765d7bb157c1c600c0, NAME => 'test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-16T05:54:03,471 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,472 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,472 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,472 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,473 INFO [StoreOpener-6ba2670ce022e672b048093a692f3cb8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ba2670ce022e672b048093a692f3cb8 columnFamilyName family 2024-11-16T05:54:03,474 DEBUG [StoreOpener-6ba2670ce022e672b048093a692f3cb8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67d7f5593e349615c92abcb96869f4ba, server=3456ee6a3164,42285,1731736440166 because future has completed 2024-11-16T05:54:03,474 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe. 2024-11-16T05:54:03,475 INFO [StoreOpener-6ba2670ce022e672b048093a692f3cb8-1 {}] regionserver.HStore(327): Store=6ba2670ce022e672b048093a692f3cb8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,475 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,475 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 0760749bb2566c6d8e6b56f164a53ebe, NAME => 'test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-16T05:54:03,475 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,475 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,476 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,476 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,476 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,476 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,479 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7df86faabb66872de7c73da2806f6ac3, server=3456ee6a3164,37837,1731736440211 because future has completed 2024-11-16T05:54:03,481 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,481 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,482 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,37837,1731736440211, table=test, region=7df86faabb66872de7c73da2806f6ac3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:54:03,485 INFO [StoreOpener-fa4cd934bc5e4b765d7bb157c1c600c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-16T05:54:03,488 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 67d7f5593e349615c92abcb96869f4ba, server=3456ee6a3164,42285,1731736440166 in 276 msec 2024-11-16T05:54:03,488 INFO [StoreOpener-fa4cd934bc5e4b765d7bb157c1c600c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa4cd934bc5e4b765d7bb157c1c600c0 columnFamilyName family 2024-11-16T05:54:03,488 DEBUG [StoreOpener-fa4cd934bc5e4b765d7bb157c1c600c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,489 INFO [StoreOpener-fa4cd934bc5e4b765d7bb157c1c600c0-1 {}] regionserver.HStore(327): Store=fa4cd934bc5e4b765d7bb157c1c600c0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,490 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,490 INFO [StoreOpener-0760749bb2566c6d8e6b56f164a53ebe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,491 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,492 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,493 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=67d7f5593e349615c92abcb96869f4ba, ASSIGN in 456 msec 2024-11-16T05:54:03,493 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,494 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/6ba2670ce022e672b048093a692f3cb8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,494 INFO [StoreOpener-0760749bb2566c6d8e6b56f164a53ebe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0760749bb2566c6d8e6b56f164a53ebe columnFamilyName family 2024-11-16T05:54:03,495 DEBUG [StoreOpener-0760749bb2566c6d8e6b56f164a53ebe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,495 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 6ba2670ce022e672b048093a692f3cb8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73719344, jitterRate=0.09850382804870605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,495 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:03,495 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 6ba2670ce022e672b048093a692f3cb8: Running coprocessor pre-open hook at 1731736443465Writing region info on filesystem at 1731736443465Initializing all the Stores at 1731736443468 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443468Cleaning up temporary data from old regions at 1731736443481 (+13 ms)Running coprocessor post-open hooks at 1731736443495 (+14 ms)Region opened successfully at 1731736443495 2024-11-16T05:54:03,495 INFO [StoreOpener-0760749bb2566c6d8e6b56f164a53ebe-1 {}] regionserver.HStore(327): Store=0760749bb2566c6d8e6b56f164a53ebe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,496 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,497 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8., pid=44, masterSystemTime=1731736443360 2024-11-16T05:54:03,497 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,497 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,498 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,499 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,499 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,501 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:03,501 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:03,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=21 2024-11-16T05:54:03,501 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:03,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 7df86faabb66872de7c73da2806f6ac3, server=3456ee6a3164,37837,1731736440211 in 230 msec 2024-11-16T05:54:03,501 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => c342bb403770b75ff0a582854ab71fbe, NAME => 'test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-16T05:54:03,502 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,502 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,502 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,502 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,503 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7df86faabb66872de7c73da2806f6ac3, ASSIGN in 469 msec 2024-11-16T05:54:03,506 INFO [StoreOpener-c342bb403770b75ff0a582854ab71fbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,506 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,506 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/fa4cd934bc5e4b765d7bb157c1c600c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,507 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened fa4cd934bc5e4b765d7bb157c1c600c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61307110, jitterRate=-0.08645287156105042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,507 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:03,507 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/0760749bb2566c6d8e6b56f164a53ebe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,508 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for fa4cd934bc5e4b765d7bb157c1c600c0: Running coprocessor pre-open hook at 1731736443472Writing region info on filesystem at 1731736443472Initializing all the Stores at 1731736443474 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443475 (+1 ms)Cleaning up temporary data from old regions at 1731736443493 (+18 ms)Running coprocessor post-open hooks at 1731736443507 (+14 ms)Region opened successfully at 1731736443507 2024-11-16T05:54:03,509 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0., pid=51, masterSystemTime=1731736443408 2024-11-16T05:54:03,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ba2670ce022e672b048093a692f3cb8, server=3456ee6a3164,42285,1731736440166 because future has completed 2024-11-16T05:54:03,510 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 0760749bb2566c6d8e6b56f164a53ebe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70307957, jitterRate=0.04767020046710968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,510 INFO [StoreOpener-c342bb403770b75ff0a582854ab71fbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c342bb403770b75ff0a582854ab71fbe columnFamilyName family 2024-11-16T05:54:03,510 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0760749bb2566c6d8e6b56f164a53ebe 2024-11-16T05:54:03,510 DEBUG [StoreOpener-c342bb403770b75ff0a582854ab71fbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,510 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 0760749bb2566c6d8e6b56f164a53ebe: Running coprocessor pre-open hook at 1731736443476Writing region info on filesystem at 1731736443476Initializing all the Stores at 1731736443478 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443478Cleaning up temporary data from old regions at 1731736443500 (+22 ms)Running coprocessor post-open hooks at 1731736443510 (+10 ms)Region opened successfully at 1731736443510 2024-11-16T05:54:03,512 INFO [StoreOpener-c342bb403770b75ff0a582854ab71fbe-1 {}] regionserver.HStore(327): Store=c342bb403770b75ff0a582854ab71fbe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,512 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe., pid=46, masterSystemTime=1731736443457 2024-11-16T05:54:03,512 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,512 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:03,513 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:03,513 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:03,513 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => ceb8a4dcf270b48605e90af8f3b8eb2c, NAME => 'test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-16T05:54:03,513 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,514 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,514 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,514 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,515 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,515 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,516 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,516 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,516 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe. 2024-11-16T05:54:03,517 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,vvv,1731736442349.0760749bb2566c6d8e6b56f164a53ebe. 2024-11-16T05:54:03,517 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:03,517 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => d63eb285e07e85739c793802fc7058c2, NAME => 'test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-16T05:54:03,517 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,517 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,518 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,518 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,519 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=8 2024-11-16T05:54:03,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ba2670ce022e672b048093a692f3cb8, server=3456ee6a3164,42285,1731736440166 in 231 msec 2024-11-16T05:54:03,522 INFO [StoreOpener-d63eb285e07e85739c793802fc7058c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:54:03,523 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/c342bb403770b75ff0a582854ab71fbe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,523 INFO [StoreOpener-ceb8a4dcf270b48605e90af8f3b8eb2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6ba2670ce022e672b048093a692f3cb8, ASSIGN in 488 msec 2024-11-16T05:54:03,524 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened c342bb403770b75ff0a582854ab71fbe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63119439, jitterRate=-0.059447064995765686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,524 INFO [StoreOpener-d63eb285e07e85739c793802fc7058c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d63eb285e07e85739c793802fc7058c2 columnFamilyName family 2024-11-16T05:54:03,524 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:03,524 DEBUG [StoreOpener-d63eb285e07e85739c793802fc7058c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,525 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for c342bb403770b75ff0a582854ab71fbe: Running coprocessor pre-open hook at 1731736443502Writing region info on filesystem at 1731736443502Initializing all the Stores at 1731736443504 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443504Cleaning up temporary data from old regions at 1731736443516 (+12 ms)Running coprocessor post-open hooks at 1731736443524 (+8 ms)Region opened successfully at 1731736443524 2024-11-16T05:54:03,525 INFO [StoreOpener-d63eb285e07e85739c793802fc7058c2-1 {}] regionserver.HStore(327): Store=d63eb285e07e85739c793802fc7058c2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,525 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,526 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,528 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,529 INFO [StoreOpener-ceb8a4dcf270b48605e90af8f3b8eb2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceb8a4dcf270b48605e90af8f3b8eb2c columnFamilyName family 2024-11-16T05:54:03,529 DEBUG [StoreOpener-ceb8a4dcf270b48605e90af8f3b8eb2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,36299,1731736440080 because future has completed 2024-11-16T05:54:03,529 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe., pid=36, masterSystemTime=1731736443360 2024-11-16T05:54:03,529 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,529 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,530 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,530 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,531 INFO [StoreOpener-ceb8a4dcf270b48605e90af8f3b8eb2c-1 {}] regionserver.HStore(327): Store=ceb8a4dcf270b48605e90af8f3b8eb2c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,531 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,532 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa4cd934bc5e4b765d7bb157c1c600c0, server=3456ee6a3164,37837,1731736440211 because future has completed 2024-11-16T05:54:03,534 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:03,534 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,534 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:03,534 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,534 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:03,535 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,535 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => e2536762964ceee99f3f786e0dad8403, NAME => 'test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-16T05:54:03,535 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,535 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,535 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,535 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,535 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,537 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,539 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d63eb285e07e85739c793802fc7058c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,539 INFO [StoreOpener-e2536762964ceee99f3f786e0dad8403-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=26 2024-11-16T05:54:03,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,36299,1731736440080 in 231 msec 2024-11-16T05:54:03,541 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened d63eb285e07e85739c793802fc7058c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72364926, jitterRate=0.0783214271068573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,541 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:03,542 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for d63eb285e07e85739c793802fc7058c2: Running coprocessor pre-open hook at 1731736443518Writing region info on filesystem at 1731736443518Initializing all the Stores at 1731736443519 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443519Cleaning up temporary data from old regions at 1731736443530 (+11 ms)Running coprocessor post-open hooks at 1731736443541 (+11 ms)Region opened successfully at 1731736443542 (+1 ms) 2024-11-16T05:54:03,542 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/ceb8a4dcf270b48605e90af8f3b8eb2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,543 INFO [StoreOpener-e2536762964ceee99f3f786e0dad8403-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2536762964ceee99f3f786e0dad8403 columnFamilyName family 2024-11-16T05:54:03,543 DEBUG [StoreOpener-e2536762964ceee99f3f786e0dad8403-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,544 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2., pid=54, masterSystemTime=1731736443457 2024-11-16T05:54:03,544 INFO [StoreOpener-e2536762964ceee99f3f786e0dad8403-1 {}] regionserver.HStore(327): Store=e2536762964ceee99f3f786e0dad8403/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,544 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened ceb8a4dcf270b48605e90af8f3b8eb2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61049802, jitterRate=-0.09028705954551697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,544 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:03,544 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,544 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for ceb8a4dcf270b48605e90af8f3b8eb2c: Running coprocessor pre-open hook at 1731736443514Writing region info on filesystem at 1731736443514Initializing all the Stores at 1731736443515 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443515Cleaning up temporary data from old regions at 1731736443535 (+20 ms)Running coprocessor post-open hooks at 1731736443544 (+9 ms)Region opened successfully at 1731736443544 2024-11-16T05:54:03,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN in 509 msec 2024-11-16T05:54:03,546 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,546 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c., pid=52, masterSystemTime=1731736443408 2024-11-16T05:54:03,546 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,547 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,547 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:03,548 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,548 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:03,548 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:03,548 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => bf20fbe8271ec5dfab659b24954f2df5, NAME => 'test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-16T05:54:03,549 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,549 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,549 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,549 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,550 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:03,550 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:03,550 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:03,550 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => f060fbfb898c418effc5c41d59bdb94f, NAME => 'test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-16T05:54:03,551 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,551 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,551 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,551 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,552 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,553 INFO [StoreOpener-bf20fbe8271ec5dfab659b24954f2df5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,554 INFO [StoreOpener-f060fbfb898c418effc5c41d59bdb94f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=6 2024-11-16T05:54:03,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure fa4cd934bc5e4b765d7bb157c1c600c0, server=3456ee6a3164,37837,1731736440211 in 219 msec 2024-11-16T05:54:03,555 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/e2536762964ceee99f3f786e0dad8403/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,555 INFO [StoreOpener-bf20fbe8271ec5dfab659b24954f2df5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf20fbe8271ec5dfab659b24954f2df5 columnFamilyName family 2024-11-16T05:54:03,555 DEBUG [StoreOpener-bf20fbe8271ec5dfab659b24954f2df5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,556 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened e2536762964ceee99f3f786e0dad8403; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61875480, jitterRate=-0.07798349857330322}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,556 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:03,556 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for e2536762964ceee99f3f786e0dad8403: Running coprocessor pre-open hook at 1731736443535Writing region info on filesystem at 1731736443535Initializing all the Stores at 1731736443537 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443537Cleaning up temporary data from old regions at 1731736443548 (+11 ms)Running coprocessor post-open hooks at 1731736443556 (+8 ms)Region opened successfully at 1731736443556 2024-11-16T05:54:03,557 INFO [StoreOpener-f060fbfb898c418effc5c41d59bdb94f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f060fbfb898c418effc5c41d59bdb94f columnFamilyName family 2024-11-16T05:54:03,558 DEBUG [StoreOpener-f060fbfb898c418effc5c41d59bdb94f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,558 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403., pid=35, masterSystemTime=1731736443360 2024-11-16T05:54:03,558 INFO [StoreOpener-bf20fbe8271ec5dfab659b24954f2df5-1 {}] regionserver.HStore(327): Store=bf20fbe8271ec5dfab659b24954f2df5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,558 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa4cd934bc5e4b765d7bb157c1c600c0, ASSIGN in 523 msec 2024-11-16T05:54:03,559 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,559 INFO [StoreOpener-f060fbfb898c418effc5c41d59bdb94f-1 {}] regionserver.HStore(327): Store=f060fbfb898c418effc5c41d59bdb94f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,559 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,560 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,560 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,561 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,561 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,561 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,561 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,561 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:03,561 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:03,562 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:03,562 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,562 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,562 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 20aedf13a99ac07f411f959c4c4bed3f, NAME => 'test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-16T05:54:03,562 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,562 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,562 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,562 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,564 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,565 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceb8a4dcf270b48605e90af8f3b8eb2c, server=3456ee6a3164,37837,1731736440211 because future has completed 2024-11-16T05:54:03,566 INFO [StoreOpener-20aedf13a99ac07f411f959c4c4bed3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,568 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,569 INFO [StoreOpener-20aedf13a99ac07f411f959c4c4bed3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20aedf13a99ac07f411f959c4c4bed3f columnFamilyName family 2024-11-16T05:54:03,570 DEBUG [StoreOpener-20aedf13a99ac07f411f959c4c4bed3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,570 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/bf20fbe8271ec5dfab659b24954f2df5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,570 INFO [StoreOpener-20aedf13a99ac07f411f959c4c4bed3f-1 {}] regionserver.HStore(327): Store=20aedf13a99ac07f411f959c4c4bed3f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,571 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,571 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened bf20fbe8271ec5dfab659b24954f2df5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69181399, jitterRate=0.030883178114891052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,571 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:03,571 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for bf20fbe8271ec5dfab659b24954f2df5: Running coprocessor pre-open hook at 1731736443549Writing region info on filesystem at 1731736443549Initializing all the Stores at 1731736443552 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443552Cleaning up temporary data from old regions at 1731736443561 (+9 ms)Running coprocessor post-open hooks at 1731736443571 (+10 ms)Region opened successfully at 1731736443571 2024-11-16T05:54:03,572 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2536762964ceee99f3f786e0dad8403, server=3456ee6a3164,42285,1731736440166 because future has completed 2024-11-16T05:54:03,573 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,573 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5., pid=47, masterSystemTime=1731736443457 2024-11-16T05:54:03,573 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,574 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,576 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:03,576 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,576 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:03,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=10 2024-11-16T05:54:03,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure ceb8a4dcf270b48605e90af8f3b8eb2c, server=3456ee6a3164,37837,1731736440211 in 238 msec 2024-11-16T05:54:03,577 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:03,577 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => df6f5bd741b507073cb45265746143fa, NAME => 'test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-16T05:54:03,578 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,578 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,578 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,579 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,579 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/f060fbfb898c418effc5c41d59bdb94f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,579 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened f060fbfb898c418effc5c41d59bdb94f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63759288, jitterRate=-0.04991257190704346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,580 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:03,580 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for f060fbfb898c418effc5c41d59bdb94f: Running coprocessor pre-open hook at 1731736443552Writing region info on filesystem at 1731736443552Initializing all the Stores at 1731736443553 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443554 (+1 ms)Cleaning up temporary data from old regions at 1731736443562 (+8 ms)Running coprocessor post-open hooks at 1731736443580 (+18 ms)Region opened successfully at 1731736443580 2024-11-16T05:54:03,581 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f., pid=41, masterSystemTime=1731736443408 2024-11-16T05:54:03,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ceb8a4dcf270b48605e90af8f3b8eb2c, ASSIGN in 545 msec 2024-11-16T05:54:03,582 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/20aedf13a99ac07f411f959c4c4bed3f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,583 INFO [StoreOpener-df6f5bd741b507073cb45265746143fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,583 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 20aedf13a99ac07f411f959c4c4bed3f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70696569, jitterRate=0.053460970520973206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,583 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:03,583 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 20aedf13a99ac07f411f959c4c4bed3f: Running coprocessor pre-open hook at 1731736443563Writing region info on filesystem at 1731736443563Initializing all the Stores at 1731736443565 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443565Cleaning up temporary data from old regions at 1731736443574 (+9 ms)Running coprocessor post-open hooks at 1731736443583 (+9 ms)Region opened successfully at 1731736443583 2024-11-16T05:54:03,584 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,584 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:03,584 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:03,585 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:03,585 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => da031f93bd094d06fcd97a63a4675032, NAME => 'test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-16T05:54:03,585 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,585 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,586 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,586 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,588 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,42285,1731736440166, table=test, region=e2536762964ceee99f3f786e0dad8403. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:54:03,588 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f., pid=33, masterSystemTime=1731736443360 2024-11-16T05:54:03,589 INFO [StoreOpener-da031f93bd094d06fcd97a63a4675032-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,591 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,591 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:03,591 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:03,592 INFO [StoreOpener-df6f5bd741b507073cb45265746143fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df6f5bd741b507073cb45265746143fa columnFamilyName family 2024-11-16T05:54:03,592 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:03,592 DEBUG [StoreOpener-df6f5bd741b507073cb45265746143fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,592 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 83571bc94223ec9eaf85f80b241d7135, NAME => 'test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-16T05:54:03,593 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,593 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,593 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,593 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,594 INFO [StoreOpener-df6f5bd741b507073cb45265746143fa-1 {}] regionserver.HStore(327): Store=df6f5bd741b507073cb45265746143fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,594 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,595 INFO [StoreOpener-da031f93bd094d06fcd97a63a4675032-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da031f93bd094d06fcd97a63a4675032 columnFamilyName family 2024-11-16T05:54:03,595 DEBUG [StoreOpener-da031f93bd094d06fcd97a63a4675032-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,36299,1731736440080 because future has completed 2024-11-16T05:54:03,597 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,36299,1731736440080, table=test, region=bf20fbe8271ec5dfab659b24954f2df5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:54:03,597 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,597 INFO [StoreOpener-da031f93bd094d06fcd97a63a4675032-1 {}] regionserver.HStore(327): Store=da031f93bd094d06fcd97a63a4675032/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,597 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,598 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,598 INFO [StoreOpener-83571bc94223ec9eaf85f80b241d7135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,599 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,599 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,599 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure f060fbfb898c418effc5c41d59bdb94f, server=3456ee6a3164,37837,1731736440211 because future has completed 2024-11-16T05:54:03,600 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,601 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,601 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,601 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,602 INFO [StoreOpener-83571bc94223ec9eaf85f80b241d7135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83571bc94223ec9eaf85f80b241d7135 columnFamilyName family 2024-11-16T05:54:03,603 DEBUG [StoreOpener-83571bc94223ec9eaf85f80b241d7135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,603 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,603 INFO [StoreOpener-83571bc94223ec9eaf85f80b241d7135-1 {}] regionserver.HStore(327): Store=83571bc94223ec9eaf85f80b241d7135/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,604 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,606 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,606 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/df6f5bd741b507073cb45265746143fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,610 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/da031f93bd094d06fcd97a63a4675032/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,610 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=17 2024-11-16T05:54:03,610 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened df6f5bd741b507073cb45265746143fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72670637, jitterRate=0.08287687599658966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,36299,1731736440080 in 291 msec 2024-11-16T05:54:03,610 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:03,610 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for df6f5bd741b507073cb45265746143fa: Running coprocessor pre-open hook at 1731736443579Writing region info on filesystem at 1731736443579Initializing all the Stores at 1731736443580 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443580Cleaning up temporary data from old regions at 1731736443599 (+19 ms)Running coprocessor post-open hooks at 1731736443610 (+11 ms)Region opened successfully at 1731736443610 2024-11-16T05:54:03,611 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,611 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened da031f93bd094d06fcd97a63a4675032; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73286851, jitterRate=0.0920591801404953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,611 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,611 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:03,611 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for da031f93bd094d06fcd97a63a4675032: Running coprocessor pre-open hook at 1731736443586Writing region info on filesystem at 1731736443586Initializing all the Stores at 1731736443587 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443587Cleaning up temporary data from old regions at 1731736443601 (+14 ms)Running coprocessor post-open hooks at 1731736443611 (+10 ms)Region opened successfully at 1731736443611 2024-11-16T05:54:03,612 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa., pid=50, masterSystemTime=1731736443457 2024-11-16T05:54:03,612 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032., pid=45, masterSystemTime=1731736443408 2024-11-16T05:54:03,614 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN in 578 msec 2024-11-16T05:54:03,615 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:03,615 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:03,616 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:03,616 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e3c5f7060b7491b30cd6cf5617a89c2, NAME => 'test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-16T05:54:03,616 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,616 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,617 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,617 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,617 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,618 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/83571bc94223ec9eaf85f80b241d7135/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,618 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:03,618 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:03,618 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. 2024-11-16T05:54:03,618 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 52d4c19e400f7867ccce41418f251b74, NAME => 'test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-16T05:54:03,618 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 83571bc94223ec9eaf85f80b241d7135; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70114363, jitterRate=0.04478542506694794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,619 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:03,619 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 83571bc94223ec9eaf85f80b241d7135: Running coprocessor pre-open hook at 1731736443596Writing region info on filesystem at 1731736443596Initializing all the Stores at 1731736443598 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443598Cleaning up temporary data from old regions at 1731736443611 (+13 ms)Running coprocessor post-open hooks at 1731736443619 (+8 ms)Region opened successfully at 1731736443619 2024-11-16T05:54:03,619 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,619 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,619 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,619 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,619 INFO [StoreOpener-8e3c5f7060b7491b30cd6cf5617a89c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,621 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135., pid=37, masterSystemTime=1731736443360 2024-11-16T05:54:03,622 INFO [StoreOpener-8e3c5f7060b7491b30cd6cf5617a89c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e3c5f7060b7491b30cd6cf5617a89c2 columnFamilyName family 2024-11-16T05:54:03,623 DEBUG [StoreOpener-8e3c5f7060b7491b30cd6cf5617a89c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure da031f93bd094d06fcd97a63a4675032, server=3456ee6a3164,37837,1731736440211 because future has completed 2024-11-16T05:54:03,624 INFO [StoreOpener-8e3c5f7060b7491b30cd6cf5617a89c2-1 {}] regionserver.HStore(327): Store=8e3c5f7060b7491b30cd6cf5617a89c2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,624 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=df6f5bd741b507073cb45265746143fa, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,625 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,625 INFO [StoreOpener-52d4c19e400f7867ccce41418f251b74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,625 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:03,625 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:03,626 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,627 INFO [StoreOpener-52d4c19e400f7867ccce41418f251b74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52d4c19e400f7867ccce41418f251b74 columnFamilyName family 2024-11-16T05:54:03,628 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,628 DEBUG [StoreOpener-52d4c19e400f7867ccce41418f251b74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,36299,1731736440080 because future has completed 2024-11-16T05:54:03,628 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,629 INFO [StoreOpener-52d4c19e400f7867ccce41418f251b74-1 {}] regionserver.HStore(327): Store=52d4c19e400f7867ccce41418f251b74/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,629 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,629 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,630 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,630 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,631 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,632 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,632 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,633 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83571bc94223ec9eaf85f80b241d7135, server=3456ee6a3164,42285,1731736440166 because future has completed 2024-11-16T05:54:03,635 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,638 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/8e3c5f7060b7491b30cd6cf5617a89c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,639 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/52d4c19e400f7867ccce41418f251b74/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,639 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 52d4c19e400f7867ccce41418f251b74; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72817140, jitterRate=0.08505994081497192}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,639 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 8e3c5f7060b7491b30cd6cf5617a89c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69245896, jitterRate=0.031844258308410645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,640 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:03,640 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:03,640 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 8e3c5f7060b7491b30cd6cf5617a89c2: Running coprocessor pre-open hook at 1731736443617Writing region info on filesystem at 1731736443617Initializing all the Stores at 1731736443619 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443619Cleaning up temporary data from old regions at 1731736443629 (+10 ms)Running coprocessor post-open hooks at 1731736443640 (+11 ms)Region opened successfully at 1731736443640 2024-11-16T05:54:03,640 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 52d4c19e400f7867ccce41418f251b74: Running coprocessor pre-open hook at 1731736443620Writing region info on filesystem at 1731736443620Initializing all the Stores at 1731736443621 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443621Cleaning up temporary data from old regions at 1731736443632 (+11 ms)Running coprocessor post-open hooks at 1731736443640 (+8 ms)Region opened successfully at 1731736443640 2024-11-16T05:54:03,641 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2., pid=42, masterSystemTime=1731736443408 2024-11-16T05:54:03,641 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74., pid=48, masterSystemTime=1731736443457 2024-11-16T05:54:03,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=29 2024-11-16T05:54:03,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,36299,1731736440080 in 311 msec 2024-11-16T05:54:03,644 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:03,644 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:03,644 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:03,645 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => d426c5a1ddcc32067a58955e792d6dae, NAME => 'test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-16T05:54:03,645 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,646 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,646 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. 2024-11-16T05:54:03,646 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,646 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. 2024-11-16T05:54:03,646 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,647 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7. 2024-11-16T05:54:03,647 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => f391d2492a2a9b1c649bcab4e64504c7, NAME => 'test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-16T05:54:03,647 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN in 611 msec 2024-11-16T05:54:03,647 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,648 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,648 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,649 INFO [StoreOpener-d426c5a1ddcc32067a58955e792d6dae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,649 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=52d4c19e400f7867ccce41418f251b74, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,650 INFO [StoreOpener-f391d2492a2a9b1c649bcab4e64504c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,651 INFO [StoreOpener-d426c5a1ddcc32067a58955e792d6dae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d426c5a1ddcc32067a58955e792d6dae columnFamilyName family 2024-11-16T05:54:03,651 DEBUG [StoreOpener-d426c5a1ddcc32067a58955e792d6dae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,653 INFO [StoreOpener-d426c5a1ddcc32067a58955e792d6dae-1 {}] regionserver.HStore(327): Store=d426c5a1ddcc32067a58955e792d6dae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,653 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52d4c19e400f7867ccce41418f251b74, server=3456ee6a3164,36299,1731736440080 because future has completed 2024-11-16T05:54:03,654 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,654 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,655 INFO [StoreOpener-f391d2492a2a9b1c649bcab4e64504c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f391d2492a2a9b1c649bcab4e64504c7 columnFamilyName family 2024-11-16T05:54:03,655 DEBUG [StoreOpener-f391d2492a2a9b1c649bcab4e64504c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,656 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,656 INFO [StoreOpener-f391d2492a2a9b1c649bcab4e64504c7-1 {}] regionserver.HStore(327): Store=f391d2492a2a9b1c649bcab4e64504c7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,656 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,656 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,656 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e3c5f7060b7491b30cd6cf5617a89c2, server=3456ee6a3164,37837,1731736440211 because future has completed 2024-11-16T05:54:03,659 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,659 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,660 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,660 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,660 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,663 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d426c5a1ddcc32067a58955e792d6dae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,664 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,664 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened d426c5a1ddcc32067a58955e792d6dae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65004295, jitterRate=-0.03136052191257477}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,664 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:03,664 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for d426c5a1ddcc32067a58955e792d6dae: Running coprocessor pre-open hook at 1731736443647Writing region info on filesystem at 1731736443647Initializing all the Stores at 1731736443648 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443648Cleaning up temporary data from old regions at 1731736443656 (+8 ms)Running coprocessor post-open hooks at 1731736443664 (+8 ms)Region opened successfully at 1731736443664 2024-11-16T05:54:03,665 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae., pid=38, masterSystemTime=1731736443408 2024-11-16T05:54:03,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=20 2024-11-16T05:54:03,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 52d4c19e400f7867ccce41418f251b74, server=3456ee6a3164,36299,1731736440080 in 345 msec 2024-11-16T05:54:03,668 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/f391d2492a2a9b1c649bcab4e64504c7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,668 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:03,668 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:03,668 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened f391d2492a2a9b1c649bcab4e64504c7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67474297, jitterRate=0.005445376038551331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,669 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f391d2492a2a9b1c649bcab4e64504c7 2024-11-16T05:54:03,669 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for f391d2492a2a9b1c649bcab4e64504c7: Running coprocessor pre-open hook at 1731736443648Writing region info on filesystem at 1731736443648Initializing all the Stores at 1731736443650 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443650Cleaning up temporary data from old regions at 1731736443660 (+10 ms)Running coprocessor post-open hooks at 1731736443669 (+9 ms)Region opened successfully at 1731736443669 2024-11-16T05:54:03,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN in 633 msec 2024-11-16T05:54:03,670 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7., pid=53, masterSystemTime=1731736443457 2024-11-16T05:54:03,671 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:03,672 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7. 2024-11-16T05:54:03,673 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,kkk,1731736442349.f391d2492a2a9b1c649bcab4e64504c7. 2024-11-16T05:54:03,673 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15. 2024-11-16T05:54:03,673 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a7892b4caedb942fa23cc8a2f9d3b15, NAME => 'test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-16T05:54:03,673 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,674 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,674 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,674 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure d426c5a1ddcc32067a58955e792d6dae, server=3456ee6a3164,37837,1731736440211 because future has completed 2024-11-16T05:54:03,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,676 INFO [StoreOpener-6a7892b4caedb942fa23cc8a2f9d3b15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,678 INFO [StoreOpener-6a7892b4caedb942fa23cc8a2f9d3b15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a7892b4caedb942fa23cc8a2f9d3b15 columnFamilyName family 2024-11-16T05:54:03,678 DEBUG [StoreOpener-6a7892b4caedb942fa23cc8a2f9d3b15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,36299,1731736440080 because future has completed 2024-11-16T05:54:03,679 INFO [StoreOpener-6a7892b4caedb942fa23cc8a2f9d3b15-1 {}] regionserver.HStore(327): Store=6a7892b4caedb942fa23cc8a2f9d3b15/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,679 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,680 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,681 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,682 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,682 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=30 2024-11-16T05:54:03,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure d426c5a1ddcc32067a58955e792d6dae, server=3456ee6a3164,37837,1731736440211 in 426 msec 2024-11-16T05:54:03,684 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d426c5a1ddcc32067a58955e792d6dae, ASSIGN in 652 msec 2024-11-16T05:54:03,689 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/6a7892b4caedb942fa23cc8a2f9d3b15/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,690 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 6a7892b4caedb942fa23cc8a2f9d3b15; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74157408, jitterRate=0.10503149032592773}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,690 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6a7892b4caedb942fa23cc8a2f9d3b15 2024-11-16T05:54:03,690 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 6a7892b4caedb942fa23cc8a2f9d3b15: Running coprocessor pre-open hook at 1731736443674Writing region info on filesystem at 1731736443674Initializing all the Stores at 1731736443675 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443675Cleaning up temporary data from old regions at 1731736443682 (+7 ms)Running coprocessor post-open hooks at 1731736443690 (+8 ms)Region opened successfully at 1731736443690 2024-11-16T05:54:03,692 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,42285,1731736440166, table=test, region=83571bc94223ec9eaf85f80b241d7135. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:54:03,692 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,42285,1731736440166, table=test, region=e2536762964ceee99f3f786e0dad8403. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:54:03,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=15 2024-11-16T05:54:03,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,36299,1731736440080 in 348 msec 2024-11-16T05:54:03,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN in 664 msec 2024-11-16T05:54:03,699 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15., pid=55, masterSystemTime=1731736443457 2024-11-16T05:54:03,700 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,37837,1731736440211, table=test, region=da031f93bd094d06fcd97a63a4675032. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:54:03,700 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,37837,1731736440211, table=test, region=f060fbfb898c418effc5c41d59bdb94f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:54:03,700 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,37837,1731736440211, table=test, region=8e3c5f7060b7491b30cd6cf5617a89c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:54:03,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=27 2024-11-16T05:54:03,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e3c5f7060b7491b30cd6cf5617a89c2, server=3456ee6a3164,37837,1731736440211 in 428 msec 2024-11-16T05:54:03,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8e3c5f7060b7491b30cd6cf5617a89c2, ASSIGN in 676 msec 2024-11-16T05:54:03,715 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15. 2024-11-16T05:54:03,715 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15. 2024-11-16T05:54:03,715 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:03,715 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c48229c22c1b63769411754c30d2dfe, NAME => 'test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-16T05:54:03,716 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,716 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,716 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,716 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=11 2024-11-16T05:54:03,718 INFO [StoreOpener-3c48229c22c1b63769411754c30d2dfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 83571bc94223ec9eaf85f80b241d7135, server=3456ee6a3164,42285,1731736440166 in 466 msec 2024-11-16T05:54:03,720 INFO [StoreOpener-3c48229c22c1b63769411754c30d2dfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c48229c22c1b63769411754c30d2dfe columnFamilyName family 2024-11-16T05:54:03,720 DEBUG [StoreOpener-3c48229c22c1b63769411754c30d2dfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,721 INFO [StoreOpener-3c48229c22c1b63769411754c30d2dfe-1 {}] regionserver.HStore(327): Store=3c48229c22c1b63769411754c30d2dfe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,721 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=83571bc94223ec9eaf85f80b241d7135, ASSIGN in 686 msec 2024-11-16T05:54:03,722 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,723 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,723 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,723 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,723 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,726 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,36299,1731736440080 because future has completed 2024-11-16T05:54:03,729 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/3c48229c22c1b63769411754c30d2dfe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,730 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 3c48229c22c1b63769411754c30d2dfe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65755318, jitterRate=-0.02016940712928772}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,730 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:03,730 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 3c48229c22c1b63769411754c30d2dfe: Running coprocessor pre-open hook at 1731736443716Writing region info on filesystem at 1731736443716Initializing all the Stores at 1731736443717 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443717Cleaning up temporary data from old regions at 1731736443723 (+6 ms)Running coprocessor post-open hooks at 1731736443730 (+7 ms)Region opened successfully at 1731736443730 2024-11-16T05:54:03,731 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe., pid=56, masterSystemTime=1731736443457 2024-11-16T05:54:03,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=14 2024-11-16T05:54:03,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure da031f93bd094d06fcd97a63a4675032, server=3456ee6a3164,37837,1731736440211 in 431 msec 2024-11-16T05:54:03,734 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:03,734 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:03,734 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. 2024-11-16T05:54:03,734 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => a62176f809b89002f2152e96ba4bef24, NAME => 'test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-16T05:54:03,735 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,735 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:03,735 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,735 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=da031f93bd094d06fcd97a63a4675032, ASSIGN in 700 msec 2024-11-16T05:54:03,737 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=3c48229c22c1b63769411754c30d2dfe, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,737 INFO [StoreOpener-a62176f809b89002f2152e96ba4bef24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,739 INFO [StoreOpener-a62176f809b89002f2152e96ba4bef24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a62176f809b89002f2152e96ba4bef24 columnFamilyName family 2024-11-16T05:54:03,739 DEBUG [StoreOpener-a62176f809b89002f2152e96ba4bef24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:03,740 INFO [StoreOpener-a62176f809b89002f2152e96ba4bef24-1 {}] regionserver.HStore(327): Store=a62176f809b89002f2152e96ba4bef24/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:03,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,36299,1731736440080 because future has completed 2024-11-16T05:54:03,740 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,742 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,742 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,743 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,743 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=9 2024-11-16T05:54:03,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,36299,1731736440080 in 388 msec 2024-11-16T05:54:03,745 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN in 713 msec 2024-11-16T05:54:03,749 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/a62176f809b89002f2152e96ba4bef24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:54:03,750 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened a62176f809b89002f2152e96ba4bef24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72385006, jitterRate=0.07862064242362976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:03,750 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:03,750 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for a62176f809b89002f2152e96ba4bef24: Running coprocessor pre-open hook at 1731736443735Writing region info on filesystem at 1731736443735Initializing all the Stores at 1731736443736 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736443737 (+1 ms)Cleaning up temporary data from old regions at 1731736443743 (+6 ms)Running coprocessor post-open hooks at 1731736443750 (+7 ms)Region opened successfully at 1731736443750 2024-11-16T05:54:03,751 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24., pid=49, masterSystemTime=1731736443457 2024-11-16T05:54:03,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=12 2024-11-16T05:54:03,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,36299,1731736440080 in 394 msec 2024-11-16T05:54:03,754 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. 2024-11-16T05:54:03,754 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. 2024-11-16T05:54:03,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN in 721 msec 2024-11-16T05:54:03,758 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=a62176f809b89002f2152e96ba4bef24, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,36299,1731736440080 because future has completed 2024-11-16T05:54:03,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=24 2024-11-16T05:54:03,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure f060fbfb898c418effc5c41d59bdb94f, server=3456ee6a3164,37837,1731736440211 in 491 msec 2024-11-16T05:54:03,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f060fbfb898c418effc5c41d59bdb94f, ASSIGN in 732 msec 2024-11-16T05:54:03,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=23 2024-11-16T05:54:03,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,36299,1731736440080 in 448 msec 2024-11-16T05:54:03,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN in 739 msec 2024-11-16T05:54:03,775 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20aedf13a99ac07f411f959c4c4bed3f, server=3456ee6a3164,42285,1731736440166 because future has completed 2024-11-16T05:54:03,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=22 2024-11-16T05:54:03,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure e2536762964ceee99f3f786e0dad8403, server=3456ee6a3164,42285,1731736440166 in 548 msec 2024-11-16T05:54:03,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e2536762964ceee99f3f786e0dad8403, ASSIGN in 749 msec 2024-11-16T05:54:03,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-11-16T05:54:03,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 20aedf13a99ac07f411f959c4c4bed3f, server=3456ee6a3164,42285,1731736440166 in 565 msec 2024-11-16T05:54:03,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=20aedf13a99ac07f411f959c4c4bed3f, ASSIGN in 756 msec 2024-11-16T05:54:03,793 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:03,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d63eb285e07e85739c793802fc7058c2, server=3456ee6a3164,36299,1731736440080 because future has completed 2024-11-16T05:54:03,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:03,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure c342bb403770b75ff0a582854ab71fbe, server=3456ee6a3164,42285,1731736440166 because future has completed 2024-11-16T05:54:03,802 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,42285,1731736440166, table=test, region=c342bb403770b75ff0a582854ab71fbe. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:54:03,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=7 2024-11-16T05:54:03,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure d63eb285e07e85739c793802fc7058c2, server=3456ee6a3164,36299,1731736440080 in 452 msec 2024-11-16T05:54:03,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, ASSIGN in 773 msec 2024-11-16T05:54:03,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=25 2024-11-16T05:54:03,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure c342bb403770b75ff0a582854ab71fbe, server=3456ee6a3164,42285,1731736440166 in 572 msec 2024-11-16T05:54:03,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=4 2024-11-16T05:54:03,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c342bb403770b75ff0a582854ab71fbe, ASSIGN in 781 msec 2024-11-16T05:54:03,819 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:54:03,820 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736443820"}]},"ts":"1731736443820"} 2024-11-16T05:54:03,823 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-16T05:54:03,824 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:54:03,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.4650 sec 2024-11-16T05:54:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:54:04,535 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-16T05:54:04,535 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-16T05:54:04,537 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:54:04,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-16T05:54:04,561 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:54:04,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-16T05:54:04,562 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:54:04,569 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-16T05:54:04,595 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-16T05:54:04,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:54:04,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:54:04,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:54:04,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:54:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d63eb285e07e85739c793802fc7058c2 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6a7892b4caedb942fa23cc8a2f9d3b15 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3c48229c22c1b63769411754c30d2dfe changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f391d2492a2a9b1c649bcab4e64504c7 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bf20fbe8271ec5dfab659b24954f2df5 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 52d4c19e400f7867ccce41418f251b74 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a62176f809b89002f2152e96ba4bef24 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0760749bb2566c6d8e6b56f164a53ebe changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region df6f5bd741b507073cb45265746143fa changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fa4cd934bc5e4b765d7bb157c1c600c0 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ceb8a4dcf270b48605e90af8f3b8eb2c changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region da031f93bd094d06fcd97a63a4675032 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fdf8bdb7617bd8bb875a900a673e25fc changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7df86faabb66872de7c73da2806f6ac3 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f060fbfb898c418effc5c41d59bdb94f changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8e3c5f7060b7491b30cd6cf5617a89c2 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d426c5a1ddcc32067a58955e792d6dae changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1309086cc4268bb14438bcc29f5b63ff changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6ba2670ce022e672b048093a692f3cb8 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 83571bc94223ec9eaf85f80b241d7135 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 20aedf13a99ac07f411f959c4c4bed3f changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 83edddaeaac4b866615839a3e7d98099 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ac4bc85feea9c2820d87cf4b219cb17c changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e2536762964ceee99f3f786e0dad8403 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c342bb403770b75ff0a582854ab71fbe changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 67d7f5593e349615c92abcb96869f4ba changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:54:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-16T05:54:04,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-16T05:54:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-16T05:54:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:54:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-11-16T05:54:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(303): server 1 has 8 regions 2024-11-16T05:54:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-16T05:54:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:54:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:54:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T05:54:04,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:54:04,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:54:04,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T05:54:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-16T05:54:04,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-16T05:54:04,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-16T05:54:05,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 513 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-16T05:54:05,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-16T05:54:05,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.HMaster(2172): balance hri=d63eb285e07e85739c793802fc7058c2, source=3456ee6a3164,36299,1731736440080, destination=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:05,201 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:05,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=d63eb285e07e85739c793802fc7058c2, regionState=CLOSING, regionLocation=3456ee6a3164,36299,1731736440080 2024-11-16T05:54:05,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE because future has completed 2024-11-16T05:54:05,205 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T05:54:05,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure d63eb285e07e85739c793802fc7058c2, server=3456ee6a3164,36299,1731736440080}] 2024-11-16T05:54:05,364 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,364 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T05:54:05,365 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing d63eb285e07e85739c793802fc7058c2, disabling compactions & flushes 2024-11-16T05:54:05,365 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:05,365 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:05,365 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. after waiting 0 ms 2024-11-16T05:54:05,365 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:05,371 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d63eb285e07e85739c793802fc7058c2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:05,373 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:05,374 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for d63eb285e07e85739c793802fc7058c2: Waiting for close lock at 1731736445365Running coprocessor pre-close hooks at 1731736445365Disabling compacts and flushes for region at 1731736445365Disabling writes for close at 1731736445365Writing region close event to WAL at 1731736445366 (+1 ms)Running coprocessor post-close hooks at 1731736445372 (+6 ms)Closed at 1731736445373 (+1 ms) 2024-11-16T05:54:05,374 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding d63eb285e07e85739c793802fc7058c2 move to 3456ee6a3164,37837,1731736440211 record at close sequenceid=2 2024-11-16T05:54:05,377 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,378 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=d63eb285e07e85739c793802fc7058c2, regionState=CLOSED 2024-11-16T05:54:05,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure d63eb285e07e85739c793802fc7058c2, server=3456ee6a3164,36299,1731736440080 because future has completed 2024-11-16T05:54:05,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-16T05:54:05,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure d63eb285e07e85739c793802fc7058c2, server=3456ee6a3164,36299,1731736440080 in 176 msec 2024-11-16T05:54:05,385 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE; state=CLOSED, location=3456ee6a3164,37837,1731736440211; forceNewPlan=false, retain=false 2024-11-16T05:54:05,536 INFO [3456ee6a3164:40501 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T05:54:05,536 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=d63eb285e07e85739c793802fc7058c2, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:05,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE because future has completed 2024-11-16T05:54:05,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure d63eb285e07e85739c793802fc7058c2, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:05,700 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:05,700 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => d63eb285e07e85739c793802fc7058c2, NAME => 'test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-16T05:54:05,701 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,701 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:05,701 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,701 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,703 INFO [StoreOpener-d63eb285e07e85739c793802fc7058c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,704 INFO [StoreOpener-d63eb285e07e85739c793802fc7058c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d63eb285e07e85739c793802fc7058c2 columnFamilyName family 2024-11-16T05:54:05,704 DEBUG [StoreOpener-d63eb285e07e85739c793802fc7058c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:05,705 INFO [StoreOpener-d63eb285e07e85739c793802fc7058c2-1 {}] regionserver.HStore(327): Store=d63eb285e07e85739c793802fc7058c2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:05,705 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,706 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,708 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,709 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,709 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,712 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,713 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened d63eb285e07e85739c793802fc7058c2; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59081223, jitterRate=-0.1196211725473404}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:05,713 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:05,713 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for d63eb285e07e85739c793802fc7058c2: Running coprocessor pre-open hook at 1731736445701Writing region info on filesystem at 1731736445701Initializing all the Stores at 1731736445702 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736445702Cleaning up temporary data from old regions at 1731736445709 (+7 ms)Running coprocessor post-open hooks at 1731736445713 (+4 ms)Region opened successfully at 1731736445713 2024-11-16T05:54:05,714 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2., pid=59, masterSystemTime=1731736445695 2024-11-16T05:54:05,717 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:05,717 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:05,718 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:05,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure d63eb285e07e85739c793802fc7058c2, server=3456ee6a3164,37837,1731736440211 because future has completed 2024-11-16T05:54:05,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-16T05:54:05,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure d63eb285e07e85739c793802fc7058c2, server=3456ee6a3164,37837,1731736440211 in 181 msec 2024-11-16T05:54:05,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE in 527 msec 2024-11-16T05:54:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-16T05:54:05,804 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:54:05,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-16T05:54:05,833 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=380, OpenFileDescriptor=638, MaxFileDescriptor=1048576, SystemLoadAverage=79, ProcessCount=11, AvailableMemoryMB=3449 2024-11-16T05:54:05,836 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-16T05:54:05,837 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-16T05:54:05,837 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 3456ee6a3164,36299,1731736440080 2024-11-16T05:54:05,838 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 3456ee6a3164,36299,1731736440080: Simulated kill ***** 2024-11-16T05:54:05,838 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T05:54:05,840 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T05:54:05,842 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T05:54:05,843 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T05:54:05,843 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T05:54:05,843 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 359909632 }, "NonHeapMemoryUsage": { "committed": 154861568, "init": 7667712, "max": -1, "used": 152455784 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "3456ee6a3164" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "3456ee6a3164" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "3456ee6a3164" } ] } 2024-11-16T05:54:05,846 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40501 {}] master.MasterRpcServices(700): 3456ee6a3164,36299,1731736440080 reported a fatal error: ***** ABORTING region server 3456ee6a3164,36299,1731736440080: Simulated kill ***** 2024-11-16T05:54:05,848 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,36299,1731736440080' ***** 2024-11-16T05:54:05,848 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-16T05:54:05,848 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:54:05,849 INFO [RS:0;3456ee6a3164:36299 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T05:54:05,849 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:54:05,849 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:54:05,849 INFO [RS:0;3456ee6a3164:36299 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T05:54:05,849 INFO [RS:0;3456ee6a3164:36299 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:54:05,849 INFO [RS:0;3456ee6a3164:36299 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3456ee6a3164:36299. 2024-11-16T05:54:05,849 DEBUG [RS:0;3456ee6a3164:36299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:54:05,853 DEBUG [RS:0;3456ee6a3164:36299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:05,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741836_1012 (size=33749) 2024-11-16T05:54:05,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741836_1012 (size=33749) 2024-11-16T05:54:05,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741836_1012 (size=33749) 2024-11-16T05:54:05,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741835_1011 (size=2774) 2024-11-16T05:54:05,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741835_1011 (size=2774) 2024-11-16T05:54:05,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741835_1011 (size=2774) 2024-11-16T05:54:05,868 DEBUG [RS:0;3456ee6a3164:36299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:05,868 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:54:05,868 INFO [RS:0;3456ee6a3164:36299 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36299 2024-11-16T05:54:05,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:54:05,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,36299,1731736440080 2024-11-16T05:54:05,877 INFO [RS:0;3456ee6a3164:36299 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:54:05,879 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,36299,1731736440080] 2024-11-16T05:54:05,880 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,36299,1731736440080 already deleted, retry=false 2024-11-16T05:54:05,880 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 3456ee6a3164,36299,1731736440080 on 3456ee6a3164,40501,1731736439501 2024-11-16T05:54:05,885 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:05,889 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for 3456ee6a3164,36299,1731736440080 (carryingMeta=true) 3456ee6a3164,36299,1731736440080/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6626fed6[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T05:54:05,890 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:05,892 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:54:05,892 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true, isMeta: true 2024-11-16T05:54:05,895 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting 2024-11-16T05:54:05,897 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,36299,1731736440080 WAL count=1, meta=true 2024-11-16T05:54:05,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta}] 2024-11-16T05:54:05,906 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:05,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:05,950 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:54:05,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:54:05,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36299-0x100471864220001, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:54:05,980 INFO [RS:0;3456ee6a3164:36299 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:54:05,980 INFO [RS:0;3456ee6a3164:36299 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,36299,1731736440080; zookeeper connection closed. 2024-11-16T05:54:05,980 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@61ff36d6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@61ff36d6 2024-11-16T05:54:06,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-16T05:54:06,101 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, size=33.0 K (33749bytes) 2024-11-16T05:54:06,102 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-16T05:54:06,103 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-16T05:54:06,103 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta 2024-11-16T05:54:06,110 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta after 4ms 2024-11-16T05:54:06,119 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta took 19ms 2024-11-16T05:54:06,128 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-16T05:54:06,134 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta.temp 2024-11-16T05:54:06,138 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta so closing down 2024-11-16T05:54:06,139 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T05:54:06,139 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta.temp 2024-11-16T05:54:06,144 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T05:54:06,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741864_1040 (size=33508) 2024-11-16T05:54:06,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741864_1040 (size=33508) 2024-11-16T05:54:06,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741864_1040 (size=33508) 2024-11-16T05:54:06,155 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-16T05:54:06,158 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta.temp to hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-16T05:54:06,161 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-16T05:54:06,161 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, journal: Splitting hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, size=33.0 K (33749bytes) at 1731736446102Finishing writing output for hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta so closing down at 1731736446138 (+36 ms)Creating recovered edits writer path=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta.temp at 1731736446139 (+1 ms)3 split writer threads finished at 1731736446144 (+5 ms)Closed recovered edits writer path=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1731736446155 (+11 ms)Rename recovered edits hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta.temp to hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1731736446158 (+3 ms)Processed 62 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1731736446161 (+3 ms) 2024-11-16T05:54:06,162 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta 2024-11-16T05:54:06,163 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-16T05:54:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-16T05:54:06,172 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta to hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/oldWALs 2024-11-16T05:54:06,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-16T05:54:06,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 in 264 msec 2024-11-16T05:54:06,176 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:06,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-16T05:54:06,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 in 278 msec 2024-11-16T05:54:06,182 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting dir is empty, no logs to split. 2024-11-16T05:54:06,182 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,36299,1731736440080 WAL count=0, meta=true 2024-11-16T05:54:06,182 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3456ee6a3164,36299,1731736440080 WAL splitting is done? wals=0, meta=true 2024-11-16T05:54:06,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T05:54:06,185 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:06,186 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:06,336 DEBUG [3456ee6a3164:40501 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T05:54:06,337 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:54:06,337 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:54:06,337 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:54:06,337 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:54:06,337 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:54:06,337 INFO [3456ee6a3164:40501 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:54:06,338 INFO [3456ee6a3164:40501 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:54:06,338 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:54:06,338 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:06,340 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPENING 2024-11-16T05:54:06,470 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T05:54:06,471 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T05:54:06,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:06,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:06,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:06,832 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:06,832 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:06,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:06,832 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:06,832 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:06,991 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T05:54:06,991 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:54:06,991 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T05:54:06,993 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C42285%2C1731736440166.meta, suffix=.meta, logDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,42285,1731736440166, archiveDir=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/oldWALs, maxLogs=32 2024-11-16T05:54:07,008 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,42285,1731736440166/3456ee6a3164%2C42285%2C1731736440166.meta.1731736446995.meta, exclude list is [], retry=0 2024-11-16T05:54:07,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34749,DS-9304333b-0d3c-4c8a-9a98-d919495de0c9,DISK] 2024-11-16T05:54:07,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39925,DS-977bdc24-b9f7-4fd5-8b69-f87ba6a67efb,DISK] 2024-11-16T05:54:07,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46509,DS-76cee6b1-1fc1-4942-af99-4fd1252edb11,DISK] 2024-11-16T05:54:07,014 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,42285,1731736440166/3456ee6a3164%2C42285%2C1731736440166.meta.1731736446995.meta 2024-11-16T05:54:07,015 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42883:42883),(127.0.0.1/127.0.0.1:40917:40917),(127.0.0.1/127.0.0.1:33953:33953)] 2024-11-16T05:54:07,015 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:54:07,015 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T05:54:07,015 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T05:54:07,016 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T05:54:07,016 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T05:54:07,016 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:07,016 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T05:54:07,016 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T05:54:07,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T05:54:07,021 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:54:07,021 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:07,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:07,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T05:54:07,023 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:54:07,023 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:07,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:07,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T05:54:07,025 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:54:07,025 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:07,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:07,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T05:54:07,027 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:54:07,027 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:07,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:07,028 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:54:07,029 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740 2024-11-16T05:54:07,032 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740 2024-11-16T05:54:07,034 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-16T05:54:07,049 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-16T05:54:07,053 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-16T05:54:07,136 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/info/fd37d456ec5843e2bb237b6e621eec30 is 138, key is test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0./info:regioninfo/1731736443529/Put/seqid=0 2024-11-16T05:54:07,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741866_1042 (size=36385) 2024-11-16T05:54:07,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741866_1042 (size=36385) 2024-11-16T05:54:07,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741866_1042 (size=36385) 2024-11-16T05:54:07,150 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/info/fd37d456ec5843e2bb237b6e621eec30 2024-11-16T05:54:07,224 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/ns/1fc9f93e77bc42f1b588569bf94cd3c6 is 43, key is default/ns:d/1731736442087/Put/seqid=0 2024-11-16T05:54:07,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741867_1043 (size=5153) 2024-11-16T05:54:07,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741867_1043 (size=5153) 2024-11-16T05:54:07,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741867_1043 (size=5153) 2024-11-16T05:54:07,235 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/ns/1fc9f93e77bc42f1b588569bf94cd3c6 2024-11-16T05:54:07,264 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/table/fbf3dd6dae5243c0bbe4d8824cd4ba76 is 40, key is test/table:state/1731736443820/Put/seqid=0 2024-11-16T05:54:07,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741868_1044 (size=5165) 2024-11-16T05:54:07,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741868_1044 (size=5165) 2024-11-16T05:54:07,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741868_1044 (size=5165) 2024-11-16T05:54:07,278 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/table/fbf3dd6dae5243c0bbe4d8824cd4ba76 2024-11-16T05:54:07,288 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/info/fd37d456ec5843e2bb237b6e621eec30 as hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/info/fd37d456ec5843e2bb237b6e621eec30 2024-11-16T05:54:07,297 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/info/fd37d456ec5843e2bb237b6e621eec30, entries=273, sequenceid=64, filesize=35.5 K 2024-11-16T05:54:07,300 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/ns/1fc9f93e77bc42f1b588569bf94cd3c6 as hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/ns/1fc9f93e77bc42f1b588569bf94cd3c6 2024-11-16T05:54:07,310 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/ns/1fc9f93e77bc42f1b588569bf94cd3c6, entries=2, sequenceid=64, filesize=5.0 K 2024-11-16T05:54:07,312 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/table/fbf3dd6dae5243c0bbe4d8824cd4ba76 as hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/table/fbf3dd6dae5243c0bbe4d8824cd4ba76 2024-11-16T05:54:07,322 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/table/fbf3dd6dae5243c0bbe4d8824cd4ba76, entries=2, sequenceid=64, filesize=5.0 K 2024-11-16T05:54:07,328 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 272ms, sequenceid=64, compaction requested=false; wal=null 2024-11-16T05:54:07,330 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T05:54:07,333 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-16T05:54:07,335 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:54:07,335 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:54:07,336 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:54:07,338 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:54:07,341 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-16T05:54:07,342 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73272046, jitterRate=0.09183856844902039}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:54:07,343 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T05:54:07,343 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731736447016Writing region info on filesystem at 1731736447016Initializing all the Stores at 1731736447017 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736447017Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736447019 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736447019Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736447019Obtaining lock to block concurrent updates at 1731736447053 (+34 ms)Preparing flush snapshotting stores in 1588230740 at 1731736447053Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1731736447056 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731736447056Flushing 1588230740/info: creating writer at 1731736447057 (+1 ms)Flushing 1588230740/info: appending metadata at 1731736447122 (+65 ms)Flushing 1588230740/info: closing flushed file at 1731736447125 (+3 ms)Flushing 1588230740/ns: creating writer at 1731736447207 (+82 ms)Flushing 1588230740/ns: appending metadata at 1731736447223 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731736447223Flushing 1588230740/table: creating writer at 1731736447245 (+22 ms)Flushing 1588230740/table: appending metadata at 1731736447263 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731736447264 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18044ec5: reopening flushed file at 1731736447286 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2427256c: reopening flushed file at 1731736447299 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a5bea69: reopening flushed file at 1731736447310 (+11 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 272ms, sequenceid=64, compaction requested=false; wal=null at 1731736447328 (+18 ms)Cleaning up temporary data from old regions at 1731736447335 (+7 ms)Running coprocessor post-open hooks at 1731736447343 (+8 ms)Region opened successfully at 1731736447343 2024-11-16T05:54:07,344 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1731736446986 2024-11-16T05:54:07,347 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T05:54:07,347 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T05:54:07,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,349 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:07,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:07,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:07,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:07,350 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,350 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:07,350 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:07,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:07,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-16T05:54:07,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 in 518 msec 2024-11-16T05:54:07,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-16T05:54:07,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1700 sec 2024-11-16T05:54:07,359 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 3456ee6a3164,36299,1731736440080 had 9 regions 2024-11-16T05:54:07,360 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true, isMeta: false 2024-11-16T05:54:07,362 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,36299,1731736440080 WAL count=1, meta=false 2024-11-16T05:54:07,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542}] 2024-11-16T05:54:07,365 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:07,466 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:54:07,518 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-16T05:54:07,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42285 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-16T05:54:07,534 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542, size=2.7 K (2774bytes) 2024-11-16T05:54:07,534 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542 2024-11-16T05:54:07,535 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542 after 0ms 2024-11-16T05:54:07,538 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542 took 4ms 2024-11-16T05:54:07,542 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 0760749bb2566c6d8e6b56f164a53ebe: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-16T05:54:07,544 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for d63eb285e07e85739c793802fc7058c2: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-16T05:54:07,546 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for bf20fbe8271ec5dfab659b24954f2df5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-16T05:54:07,547 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for df6f5bd741b507073cb45265746143fa: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-16T05:54:07,549 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 52d4c19e400f7867ccce41418f251b74: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-16T05:54:07,551 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f391d2492a2a9b1c649bcab4e64504c7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-16T05:54:07,553 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 6a7892b4caedb942fa23cc8a2f9d3b15: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-16T05:54:07,555 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 3c48229c22c1b63769411754c30d2dfe: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-16T05:54:07,556 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for a62176f809b89002f2152e96ba4bef24: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-16T05:54:07,556 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542 so closing down 2024-11-16T05:54:07,557 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T05:54:07,557 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T05:54:07,557 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 18 ms; skipped=10; WAL=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-11-16T05:54:07,557 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542, journal: Splitting hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542, size=2.7 K (2774bytes) at 1731736447534Finishing writing output for hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542 so closing down at 1731736447557 (+23 ms)3 split writer threads finished at 1731736447557Processed 10 edits across 0 Regions in 18 ms; skipped=10; WAL=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1731736447557 2024-11-16T05:54:07,557 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542 2024-11-16T05:54:07,557 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-16T05:54:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40501 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-16T05:54:07,569 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting/3456ee6a3164%2C36299%2C1731736440080.1731736441542 to hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/oldWALs 2024-11-16T05:54:07,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-16T05:54:07,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 in 204 msec 2024-11-16T05:54:07,573 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-16T05:54:07,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 in 210 msec 2024-11-16T05:54:07,578 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/WALs/3456ee6a3164,36299,1731736440080-splitting dir is empty, no logs to split. 2024-11-16T05:54:07,578 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,36299,1731736440080 WAL count=0, meta=false 2024-11-16T05:54:07,578 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3456ee6a3164,36299,1731736440080 WAL splitting is done? wals=0, meta=false 2024-11-16T05:54:07,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN}] 2024-11-16T05:54:07,583 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:07,583 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:07,584 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:07,585 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:07,586 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:07,587 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:07,587 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:07,588 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:07,589 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:07,589 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:07,590 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:07,591 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:07,592 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:07,593 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:07,594 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:07,594 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:07,734 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:54:07,734 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:54:07,734 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:54:07,734 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:54:07,734 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:54:07,735 INFO [3456ee6a3164:40501 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:54:07,735 INFO [3456ee6a3164:40501 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:54:07,735 DEBUG [3456ee6a3164:40501 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:54:07,735 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,736 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,737 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,737 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:36299 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:07,737 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, regionLocation=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,738 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=df6f5bd741b507073cb45265746143fa, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,738 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:36299 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 2024-11-16T05:54:07,739 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:54:07,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1 from cache 2024-11-16T05:54:07,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, regionLocation=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,741 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:07,741 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,42285,1731736440166, seqNum=-1] 2024-11-16T05:54:07,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:36299 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 2024-11-16T05:54:07,742 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:54:07,743 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:36299 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:07,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,42285,1731736440166, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:36299 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 2024-11-16T05:54:07,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:54:07,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,42285,1731736440166, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:36299 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 2024-11-16T05:54:07,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,42285,1731736440166, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:36299 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 2024-11-16T05:54:07,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,42285,1731736440166, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:36299 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 2024-11-16T05:54:07,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,42285,1731736440166, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:36299 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 2024-11-16T05:54:07,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36299,1731736440080, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,42285,1731736440166, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:36299 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 2024-11-16T05:54:07,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:36299 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:36299 2024-11-16T05:54:07,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN because future has completed 2024-11-16T05:54:07,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:07,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN because future has completed 2024-11-16T05:54:07,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:07,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN because future has completed 2024-11-16T05:54:07,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN because future has completed 2024-11-16T05:54:07,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN because future has completed 2024-11-16T05:54:07,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN because future has completed 2024-11-16T05:54:07,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:07,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN because future has completed 2024-11-16T05:54:07,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN because future has completed 2024-11-16T05:54:07,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:07,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:07,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:07,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52d4c19e400f7867ccce41418f251b74, server=3456ee6a3164,37837,1731736440211}] 2024-11-16T05:54:07,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166}] 2024-11-16T05:54:07,903 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:07,903 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => bf20fbe8271ec5dfab659b24954f2df5, NAME => 'test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-16T05:54:07,904 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,904 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:07,904 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,904 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,906 INFO [StoreOpener-bf20fbe8271ec5dfab659b24954f2df5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,907 INFO [StoreOpener-bf20fbe8271ec5dfab659b24954f2df5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf20fbe8271ec5dfab659b24954f2df5 columnFamilyName family 2024-11-16T05:54:07,907 DEBUG [StoreOpener-bf20fbe8271ec5dfab659b24954f2df5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:07,908 INFO [StoreOpener-bf20fbe8271ec5dfab659b24954f2df5-1 {}] regionserver.HStore(327): Store=bf20fbe8271ec5dfab659b24954f2df5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:07,908 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,909 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,910 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,910 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,910 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,912 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,913 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened bf20fbe8271ec5dfab659b24954f2df5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72597894, jitterRate=0.0817929208278656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:07,914 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:07,914 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for bf20fbe8271ec5dfab659b24954f2df5: Running coprocessor pre-open hook at 1731736447904Writing region info on filesystem at 1731736447904Initializing all the Stores at 1731736447905 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736447905Cleaning up temporary data from old regions at 1731736447910 (+5 ms)Running coprocessor post-open hooks at 1731736447914 (+4 ms)Region opened successfully at 1731736447914 2024-11-16T05:54:07,915 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5., pid=82, masterSystemTime=1731736447899 2024-11-16T05:54:07,917 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:07,917 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:07,917 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:07,918 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => df6f5bd741b507073cb45265746143fa, NAME => 'test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-16T05:54:07,918 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,918 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:07,918 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,918 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,920 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-16T05:54:07,920 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,920 INFO [StoreOpener-df6f5bd741b507073cb45265746143fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,921 INFO [StoreOpener-df6f5bd741b507073cb45265746143fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df6f5bd741b507073cb45265746143fa columnFamilyName family 2024-11-16T05:54:07,921 DEBUG [StoreOpener-df6f5bd741b507073cb45265746143fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:07,920 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:07,922 INFO [StoreOpener-df6f5bd741b507073cb45265746143fa-1 {}] regionserver.HStore(327): Store=df6f5bd741b507073cb45265746143fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:07,922 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,923 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,924 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,925 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,925 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,927 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,928 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened df6f5bd741b507073cb45265746143fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65326984, jitterRate=-0.02655208110809326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:07,928 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:07,929 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for df6f5bd741b507073cb45265746143fa: Running coprocessor pre-open hook at 1731736447918Writing region info on filesystem at 1731736447918Initializing all the Stores at 1731736447919 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736447920 (+1 ms)Cleaning up temporary data from old regions at 1731736447925 (+5 ms)Running coprocessor post-open hooks at 1731736447928 (+3 ms)Region opened successfully at 1731736447928 2024-11-16T05:54:07,929 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa., pid=79, masterSystemTime=1731736447899 2024-11-16T05:54:07,932 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:07,932 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:07,932 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:07,932 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c48229c22c1b63769411754c30d2dfe, NAME => 'test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-16T05:54:07,933 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,933 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:07,931 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-16T05:54:07,933 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,933 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:07,933 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:07,934 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:07,935 INFO [StoreOpener-3c48229c22c1b63769411754c30d2dfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,936 INFO [StoreOpener-3c48229c22c1b63769411754c30d2dfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c48229c22c1b63769411754c30d2dfe columnFamilyName family 2024-11-16T05:54:07,936 DEBUG [StoreOpener-3c48229c22c1b63769411754c30d2dfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:07,936 INFO [StoreOpener-3c48229c22c1b63769411754c30d2dfe-1 {}] regionserver.HStore(327): Store=3c48229c22c1b63769411754c30d2dfe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:07,937 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,937 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,939 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,939 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,939 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,941 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,942 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 3c48229c22c1b63769411754c30d2dfe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65322319, jitterRate=-0.02662159502506256}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:07,942 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:07,942 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 3c48229c22c1b63769411754c30d2dfe: Running coprocessor pre-open hook at 1731736447933Writing region info on filesystem at 1731736447933Initializing all the Stores at 1731736447934 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736447934Cleaning up temporary data from old regions at 1731736447939 (+5 ms)Running coprocessor post-open hooks at 1731736447942 (+3 ms)Region opened successfully at 1731736447942 2024-11-16T05:54:07,943 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe., pid=75, masterSystemTime=1731736447899 2024-11-16T05:54:07,945 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:07,945 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:07,946 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. 2024-11-16T05:54:07,946 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => a62176f809b89002f2152e96ba4bef24, NAME => 'test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-16T05:54:07,946 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,946 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:07,946 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,946 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,948 INFO [StoreOpener-a62176f809b89002f2152e96ba4bef24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,949 INFO [StoreOpener-a62176f809b89002f2152e96ba4bef24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a62176f809b89002f2152e96ba4bef24 columnFamilyName family 2024-11-16T05:54:07,949 DEBUG [StoreOpener-a62176f809b89002f2152e96ba4bef24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:07,950 INFO [StoreOpener-a62176f809b89002f2152e96ba4bef24-1 {}] regionserver.HStore(327): Store=a62176f809b89002f2152e96ba4bef24/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:07,950 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,951 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,952 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,953 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,953 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,955 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:07,955 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:07,955 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:07,955 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,955 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:07,956 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:07,956 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:07,956 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:07,956 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened a62176f809b89002f2152e96ba4bef24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71687746, jitterRate=0.06823065876960754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:07,956 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a62176f809b89002f2152e96ba4bef24 2024-11-16T05:54:07,957 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for a62176f809b89002f2152e96ba4bef24: Running coprocessor pre-open hook at 1731736447946Writing region info on filesystem at 1731736447946Initializing all the Stores at 1731736447947 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736447947Cleaning up temporary data from old regions at 1731736447953 (+6 ms)Running coprocessor post-open hooks at 1731736447956 (+3 ms)Region opened successfully at 1731736447957 (+1 ms) 2024-11-16T05:54:07,957 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:07,957 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:07,958 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24., pid=78, masterSystemTime=1731736447899 2024-11-16T05:54:07,959 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:07,960 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "3456ee6a3164" port: 42285 start_code: 1731736440166 } transition { transition_code: OPENED region_info { region_id: 1731736442349 table_name { namespace: "default" qualifier: "test" } start_key: "sss" end_key: "ttt" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 78 initiating_master_active_time: 1731736440286 }; retry (#0) after 1000ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T05:54:07,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:07,966 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:07,967 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:07,967 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:07,967 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:07,967 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:07,967 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:07,967 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:07,967 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:07,967 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:07,968 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:07,970 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:07,970 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:07,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:07,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:07,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:07,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:07,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52d4c19e400f7867ccce41418f251b74, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:07,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:07,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:07,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:07,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:07,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:07,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:07,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:07,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:07,979 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:07,979 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:07,979 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:07,979 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:07,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52d4c19e400f7867ccce41418f251b74, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:07,980 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 22 msec 2024-11-16T05:54:07,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:07,982 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:07,982 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:07,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:07,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:07,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:07,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:07,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:07,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:07,986 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:07,986 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:07,986 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:07,986 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:07,986 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:07,986 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:07,986 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN to state=OFFLINE, location=null, table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15 to restore RIT 2024-11-16T05:54:07,986 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN to state=OFFLINE, location=null, table=test, region=f391d2492a2a9b1c649bcab4e64504c7 to restore RIT 2024-11-16T05:54:07,987 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736477987 2024-11-16T05:54:07,987 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:07,987 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:07,989 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:07,991 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:07,991 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52d4c19e400f7867ccce41418f251b74, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:07,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:07,991 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52d4c19e400f7867ccce41418f251b74, server=3456ee6a3164,37837,1731736440211 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,011 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. 2024-11-16T05:54:08,011 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 52d4c19e400f7867ccce41418f251b74, NAME => 'test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-16T05:54:08,012 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,012 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:08,012 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,012 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,014 INFO [StoreOpener-52d4c19e400f7867ccce41418f251b74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,016 INFO [StoreOpener-52d4c19e400f7867ccce41418f251b74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52d4c19e400f7867ccce41418f251b74 columnFamilyName family 2024-11-16T05:54:08,016 DEBUG [StoreOpener-52d4c19e400f7867ccce41418f251b74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:08,016 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,016 INFO [StoreOpener-52d4c19e400f7867ccce41418f251b74-1 {}] regionserver.HStore(327): Store=52d4c19e400f7867ccce41418f251b74/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:08,017 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,017 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,018 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,019 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,019 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,019 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,021 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,021 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,021 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,021 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,022 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,022 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,022 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 52d4c19e400f7867ccce41418f251b74; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71740547, jitterRate=0.0690174549818039}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:54:08,022 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 52d4c19e400f7867ccce41418f251b74 2024-11-16T05:54:08,022 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 52d4c19e400f7867ccce41418f251b74: Running coprocessor pre-open hook at 1731736448012Writing region info on filesystem at 1731736448012Initializing all the Stores at 1731736448014 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736448014Cleaning up temporary data from old regions at 1731736448019 (+5 ms)Running coprocessor post-open hooks at 1731736448022 (+3 ms)Region opened successfully at 1731736448022 2024-11-16T05:54:08,022 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,023 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74., pid=81, masterSystemTime=1731736448006 2024-11-16T05:54:08,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,024 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,025 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,026 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508026 2024-11-16T05:54:08,026 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568026 2024-11-16T05:54:08,026 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 38 msec 2024-11-16T05:54:08,026 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,026 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,026 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,026 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,026 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,026 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,027 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,027 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,028 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,028 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.CallRunner(138): callId: 75 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 126 connection: 172.17.0.2:35555 deadline: 1731736458024, exception=org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false 2024-11-16T05:54:08,030 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "3456ee6a3164" port: 37837 start_code: 1731736440211 } transition { transition_code: OPENED region_info { region_id: 1731736442349 table_name { namespace: "default" qualifier: "test" } start_key: "ppp" end_key: "qqq" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 81 initiating_master_active_time: 1731736440286 }; retry (#0) after 1006ms delay (Master is coming online...). org.apache.hadoop.hbase.PleaseHoldException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,035 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,037 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52d4c19e400f7867ccce41418f251b74, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,042 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,042 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,042 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,042 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,043 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,043 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,043 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,043 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,043 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,043 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52d4c19e400f7867ccce41418f251b74, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,044 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-16T05:54:08,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,045 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,045 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,048 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,049 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,050 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,050 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:08,050 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN to state=OFFLINE, location=null, table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15 to restore RIT 2024-11-16T05:54:08,050 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN to state=OFFLINE, location=null, table=test, region=f391d2492a2a9b1c649bcab4e64504c7 to restore RIT 2024-11-16T05:54:08,050 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478050 2024-11-16T05:54:08,050 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,050 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,052 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,053 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,054 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,054 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,37837,1731736440211 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,064 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,064 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508064 2024-11-16T05:54:08,064 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568064 2024-11-16T05:54:08,064 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-16T05:54:08,064 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,065 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,065 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,065 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,065 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,065 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,066 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,074 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,075 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,078 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,079 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,080 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,080 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,080 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,080 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,081 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,081 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,081 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,081 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f391d2492a2a9b1c649bcab4e64504c7, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,081 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-16T05:54:08,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,083 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,083 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,086 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,086 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,086 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,087 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,087 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,087 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,087 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,087 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,087 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:08,087 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN to state=OFFLINE, location=null, table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15 to restore RIT 2024-11-16T05:54:08,087 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN to state=OFFLINE, location=null, table=test, region=f391d2492a2a9b1c649bcab4e64504c7 to restore RIT 2024-11-16T05:54:08,088 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478088 2024-11-16T05:54:08,088 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,088 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,090 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,091 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,091 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,091 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,102 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,102 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508102 2024-11-16T05:54:08,102 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568102 2024-11-16T05:54:08,103 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-16T05:54:08,103 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,103 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,103 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,103 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,103 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,103 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,103 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,104 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,104 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,109 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,110 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,116 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,117 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,117 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,117 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,117 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,117 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,117 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,117 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,117 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,117 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,118 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,118 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure df6f5bd741b507073cb45265746143fa, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,118 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-16T05:54:08,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,119 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,119 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,122 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,122 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,122 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:08,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN to state=OFFLINE, location=null, table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15 to restore RIT 2024-11-16T05:54:08,123 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN to state=OFFLINE, location=null, table=test, region=f391d2492a2a9b1c649bcab4e64504c7 to restore RIT 2024-11-16T05:54:08,123 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478123 2024-11-16T05:54:08,123 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,123 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,125 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,126 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,126 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,127 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,135 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,135 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508135 2024-11-16T05:54:08,135 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568135 2024-11-16T05:54:08,136 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-16T05:54:08,136 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,136 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,136 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,136 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,136 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,136 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,145 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,147 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,147 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,147 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,147 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,147 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,147 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62176f809b89002f2152e96ba4bef24, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,147 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-16T05:54:08,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,148 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,148 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,152 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:08,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN to state=OFFLINE, location=null, table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15 to restore RIT 2024-11-16T05:54:08,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN to state=OFFLINE, location=null, table=test, region=f391d2492a2a9b1c649bcab4e64504c7 to restore RIT 2024-11-16T05:54:08,153 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478153 2024-11-16T05:54:08,153 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,154 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,156 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,157 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,158 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,158 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,163 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,163 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508163 2024-11-16T05:54:08,163 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568163 2024-11-16T05:54:08,163 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-16T05:54:08,163 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,164 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,164 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,164 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,164 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,164 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,164 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,164 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,165 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,168 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,169 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,173 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,173 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,173 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,174 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,175 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,175 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,175 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,175 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,175 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,175 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,175 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,175 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,175 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,175 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,175 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,175 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,176 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,176 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,176 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,176 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,176 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,176 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,176 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,176 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,176 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,176 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,176 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,176 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,176 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,176 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a7892b4caedb942fa23cc8a2f9d3b15, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-16T05:54:08,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,178 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,178 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,181 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,181 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:08,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN to state=OFFLINE, location=null, table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15 to restore RIT 2024-11-16T05:54:08,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN to state=OFFLINE, location=null, table=test, region=f391d2492a2a9b1c649bcab4e64504c7 to restore RIT 2024-11-16T05:54:08,182 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478182 2024-11-16T05:54:08,183 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,183 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,184 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,186 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,186 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,186 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,195 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,195 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508195 2024-11-16T05:54:08,195 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568195 2024-11-16T05:54:08,195 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-16T05:54:08,196 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,196 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,196 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,196 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,196 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,196 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,196 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,196 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,197 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,203 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,205 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,206 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,206 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0760749bb2566c6d8e6b56f164a53ebe, server=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,207 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-16T05:54:08,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,208 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,208 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,213 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,213 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,213 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,213 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,213 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,213 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:08,213 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN to state=OFFLINE, location=null, table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15 to restore RIT 2024-11-16T05:54:08,213 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN to state=OFFLINE, location=null, table=test, region=f391d2492a2a9b1c649bcab4e64504c7 to restore RIT 2024-11-16T05:54:08,214 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478213 2024-11-16T05:54:08,214 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,214 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,216 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,217 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,217 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,217 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,223 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,223 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508223 2024-11-16T05:54:08,223 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568223 2024-11-16T05:54:08,223 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-16T05:54:08,223 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,223 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,223 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,223 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,223 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,223 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,223 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,224 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,224 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,226 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,227 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,230 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,231 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,231 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,231 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,231 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,231 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,231 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,231 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,231 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,231 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,231 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,231 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,231 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c48229c22c1b63769411754c30d2dfe, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,231 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-16T05:54:08,232 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,232 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,232 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:08,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN to state=OFFLINE, location=null, table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15 to restore RIT 2024-11-16T05:54:08,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN to state=OFFLINE, location=null, table=test, region=f391d2492a2a9b1c649bcab4e64504c7 to restore RIT 2024-11-16T05:54:08,237 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478237 2024-11-16T05:54:08,237 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,237 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,239 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,240 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,240 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,240 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,246 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,246 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508246 2024-11-16T05:54:08,246 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568246 2024-11-16T05:54:08,246 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-16T05:54:08,247 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,247 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,247 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,247 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,247 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,247 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,247 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,247 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,248 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,252 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,253 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,254 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,254 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,254 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,254 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,254 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,254 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,254 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,254 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,254 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,254 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,254 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,254 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,254 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,254 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,254 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,254 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN 2024-11-16T05:54:08,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-16T05:54:08,255 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,256 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,256 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,256 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,260 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,260 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,261 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,261 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,261 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,261 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:08,261 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN to state=OFFLINE, location=null, table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15 to restore RIT 2024-11-16T05:54:08,261 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f391d2492a2a9b1c649bcab4e64504c7, ASSIGN to state=OFFLINE, location=null, table=test, region=f391d2492a2a9b1c649bcab4e64504c7 to restore RIT 2024-11-16T05:54:08,261 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478261 2024-11-16T05:54:08,261 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,261 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,263 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,264 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,264 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,264 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,267 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,268 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,270 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,270 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508270 2024-11-16T05:54:08,270 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568270 2024-11-16T05:54:08,270 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-16T05:54:08,270 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,270 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,270 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,270 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,270 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,270 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,270 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,270 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,271 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,273 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,274 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,275 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,275 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,275 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,275 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,275 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,275 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,276 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,276 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,276 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,277 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,277 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,277 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,277 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,277 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,277 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,277 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,277 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,277 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,277 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN 2024-11-16T05:54:08,277 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-16T05:54:08,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,278 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,278 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,281 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:08,282 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15, ASSIGN to state=OFFLINE, location=null, table=test, region=6a7892b4caedb942fa23cc8a2f9d3b15 to restore RIT 2024-11-16T05:54:08,282 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478282 2024-11-16T05:54:08,282 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,282 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,283 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,284 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,285 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,285 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,287 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,290 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,290 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508290 2024-11-16T05:54:08,290 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568290 2024-11-16T05:54:08,290 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:08,290 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,290 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,290 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,290 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,290 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,290 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,290 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,291 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,291 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,293 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,294 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,294 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,294 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,295 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,296 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,296 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,296 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,296 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,296 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,296 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,296 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN 2024-11-16T05:54:08,297 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-16T05:54:08,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,297 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,297 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,300 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,300 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,300 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,301 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=52d4c19e400f7867ccce41418f251b74, ASSIGN to state=OFFLINE, location=null, table=test, region=52d4c19e400f7867ccce41418f251b74 to restore RIT 2024-11-16T05:54:08,302 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478302 2024-11-16T05:54:08,302 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,302 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,303 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,304 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,304 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,305 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,306 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,308 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,309 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,309 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508309 2024-11-16T05:54:08,309 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568309 2024-11-16T05:54:08,309 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:08,309 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,309 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,310 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,310 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,310 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,310 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,310 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,310 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,311 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,313 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,314 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,315 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,316 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,316 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,316 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,316 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,316 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,316 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN 2024-11-16T05:54:08,316 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-16T05:54:08,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,317 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,317 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,319 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,319 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,319 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,320 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bf20fbe8271ec5dfab659b24954f2df5, ASSIGN to state=OFFLINE, location=null, table=test, region=bf20fbe8271ec5dfab659b24954f2df5 to restore RIT 2024-11-16T05:54:08,320 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478320 2024-11-16T05:54:08,320 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,320 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,322 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,323 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,323 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,323 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,328 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,328 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508328 2024-11-16T05:54:08,328 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568328 2024-11-16T05:54:08,328 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:08,328 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,328 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,328 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,328 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,328 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,329 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,329 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,330 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,334 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,334 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,334 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,334 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,335 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN 2024-11-16T05:54:08,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-16T05:54:08,336 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,336 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,336 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a62176f809b89002f2152e96ba4bef24, ASSIGN to state=OFFLINE, location=null, table=test, region=a62176f809b89002f2152e96ba4bef24 to restore RIT 2024-11-16T05:54:08,339 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478339 2024-11-16T05:54:08,339 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,339 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,341 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,342 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,342 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,343 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,347 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,347 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508347 2024-11-16T05:54:08,347 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568347 2024-11-16T05:54:08,347 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:08,347 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,347 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,347 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,347 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,347 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,348 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,348 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,353 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,353 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,353 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,353 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN 2024-11-16T05:54:08,353 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T05:54:08,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,354 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,355 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,357 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,357 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,357 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,358 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,358 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,358 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0760749bb2566c6d8e6b56f164a53ebe, ASSIGN to state=OFFLINE, location=null, table=test, region=0760749bb2566c6d8e6b56f164a53ebe to restore RIT 2024-11-16T05:54:08,358 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478358 2024-11-16T05:54:08,358 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,358 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,360 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,362 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,362 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,362 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,368 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,368 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508368 2024-11-16T05:54:08,368 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568368 2024-11-16T05:54:08,368 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-16T05:54:08,368 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,368 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,369 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,369 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,369 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,369 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,369 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,369 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,370 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,372 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,373 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,373 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,375 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,376 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,376 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN 2024-11-16T05:54:08,376 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-16T05:54:08,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,377 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,378 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,382 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,383 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=df6f5bd741b507073cb45265746143fa, ASSIGN to state=OFFLINE, location=null, table=test, region=df6f5bd741b507073cb45265746143fa to restore RIT 2024-11-16T05:54:08,383 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478383 2024-11-16T05:54:08,383 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,383 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,385 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,388 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,388 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,389 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,393 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,394 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508393 2024-11-16T05:54:08,394 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568394 2024-11-16T05:54:08,394 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-16T05:54:08,394 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,394 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,394 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,394 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,394 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,394 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,394 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,395 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,396 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,398 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,398 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,398 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,398 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,398 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,398 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,398 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,398 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,398 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,402 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,402 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN 2024-11-16T05:54:08,402 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-16T05:54:08,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,403 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,403 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,405 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,405 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,405 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,405 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c48229c22c1b63769411754c30d2dfe, ASSIGN to state=OFFLINE, location=null, table=test, region=3c48229c22c1b63769411754c30d2dfe to restore RIT 2024-11-16T05:54:08,406 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478406 2024-11-16T05:54:08,406 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,406 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,408 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,409 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,409 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,410 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,415 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,415 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508415 2024-11-16T05:54:08,415 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568415 2024-11-16T05:54:08,415 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-16T05:54:08,415 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,415 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,415 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,416 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,416 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,416 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,416 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,416 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,417 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,418 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,419 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,419 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,419 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,419 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,419 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,419 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,419 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,420 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,420 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,421 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T05:54:08,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,423 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,423 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,425 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478425 2024-11-16T05:54:08,425 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,425 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,427 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,428 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,428 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,428 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,432 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,432 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,432 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508432 2024-11-16T05:54:08,432 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568432 2024-11-16T05:54:08,432 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:08,432 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,432 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,433 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,433 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,433 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,433 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,433 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,433 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,434 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,435 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,436 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,437 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,437 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,438 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,438 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,438 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,438 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,438 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.1731736441542, worker=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,438 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T05:54:08,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,439 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,439 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,441 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478441 2024-11-16T05:54:08,441 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,441 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,443 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,444 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,444 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,444 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,447 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,448 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,448 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,448 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508448 2024-11-16T05:54:08,448 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568448 2024-11-16T05:54:08,448 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-16T05:54:08,448 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,448 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,448 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,448 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,448 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,448 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,449 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,449 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,449 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:08,451 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,452 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,452 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,452 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,452 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,452 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,452 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,452 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,452 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,452 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,453 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,453 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,453 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,42285,1731736440166 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,453 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-16T05:54:08,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,453 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,454 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,455 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,455 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,455 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,456 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478456 2024-11-16T05:54:08,456 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,456 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,457 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,458 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,459 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,459 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,463 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,463 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508463 2024-11-16T05:54:08,463 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568463 2024-11-16T05:54:08,463 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:08,463 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,463 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,463 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,463 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,464 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,464 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,464 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,464 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,464 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,465 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40501 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T05:54:08,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:08,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:08,468 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,468 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-16T05:54:08,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,469 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,469 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,471 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,471 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,471 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,472 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478472 2024-11-16T05:54:08,472 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,472 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,473 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,476 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,476 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,476 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,478 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,478 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,479 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508478 2024-11-16T05:54:08,479 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568479 2024-11-16T05:54:08,479 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-16T05:54:08,479 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,479 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,479 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,479 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,479 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,479 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,479 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,479 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,480 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:08,482 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,482 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,482 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,483 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,483 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,483 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,483 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-16T05:54:08,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,484 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,484 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,486 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478486 2024-11-16T05:54:08,486 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,486 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,488 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,489 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,489 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,489 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,494 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,494 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508494 2024-11-16T05:54:08,494 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568494 2024-11-16T05:54:08,494 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:08,494 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,494 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,494 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,494 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,494 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,494 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,494 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,494 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,495 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:08,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C36299%2C1731736440080.meta.1731736441929.meta, worker=3456ee6a3164,37837,1731736440211 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:08,498 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-16T05:54:08,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,498 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,499 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,501 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478501 2024-11-16T05:54:08,501 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,501 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,503 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,504 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-16T05:54:08,504 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,504 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,509 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,509 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508509 2024-11-16T05:54:08,509 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568509 2024-11-16T05:54:08,509 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:08,509 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:08,509 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,509 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,509 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,509 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:08,509 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:08,509 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:08,510 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:08,510 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:08,512 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:08,512 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:08,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-16T05:54:08,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63eb285e07e85739c793802fc7058c2, REOPEN/MOVE 2024-11-16T05:54:08,513 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true 2024-11-16T05:54:08,513 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:08,513 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-16T05:54:08,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=65 2024-11-16T05:54:08,514 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,42285,1731736440166, table=hbase:meta, region=1588230740 2024-11-16T05:54:08,514 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,42285,1731736440166, state=OPEN 2024-11-16T05:54:08,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:08,516 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,516 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,516 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:08,517 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736478517 2024-11-16T05:54:08,517 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:08,517 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:08,518 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-16T05:54:08,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=82, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure bf20fbe8271ec5dfab659b24954f2df5, server=3456ee6a3164,42285,1731736440166:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3456ee6a3164,36299,1731736440080, splitWal=true, meta=true exec-time=2.6370 sec 2024-11-16T05:54:08,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1309086cc4268bb14438bcc29f5b63ff, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa4cd934bc5e4b765d7bb157c1c600c0, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63eb285e07e85739c793802fc7058c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=5 2024-11-16T05:54:08,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ba2670ce022e672b048093a692f3cb8, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a7892b4caedb942fa23cc8a2f9d3b15, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ceb8a4dcf270b48605e90af8f3b8eb2c, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83571bc94223ec9eaf85f80b241d7135, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c48229c22c1b63769411754c30d2dfe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20aedf13a99ac07f411f959c4c4bed3f, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da031f93bd094d06fcd97a63a4675032, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f391d2492a2a9b1c649bcab4e64504c7, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83edddaeaac4b866615839a3e7d98099, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf20fbe8271ec5dfab659b24954f2df5, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fdf8bdb7617bd8bb875a900a673e25fc, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac4bc85feea9c2820d87cf4b219cb17c, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52d4c19e400f7867ccce41418f251b74, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7df86faabb66872de7c73da2806f6ac3, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e2536762964ceee99f3f786e0dad8403, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a62176f809b89002f2152e96ba4bef24, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f060fbfb898c418effc5c41d59bdb94f, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c342bb403770b75ff0a582854ab71fbe, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0760749bb2566c6d8e6b56f164a53ebe, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e3c5f7060b7491b30cd6cf5617a89c2, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=67d7f5593e349615c92abcb96869f4ba, regionState=OPEN, lastHost=3456ee6a3164,42285,1731736440166, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df6f5bd741b507073cb45265746143fa, regionState=OPENING, lastHost=3456ee6a3164,36299,1731736440080, regionLocation=3456ee6a3164,42285,1731736440166, openSeqNum=2 2024-11-16T05:54:08,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d426c5a1ddcc32067a58955e792d6dae, regionState=OPEN, lastHost=3456ee6a3164,37837,1731736440211, regionLocation=3456ee6a3164,37837,1731736440211, openSeqNum=2 2024-11-16T05:54:08,524 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:08,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736508524 2024-11-16T05:54:08,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736568525 2024-11-16T05:54:08,525 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:08,542 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=348 (was 380), OpenFileDescriptor=616 (was 638), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=97 (was 79) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3385 (was 3449) 2024-11-16T05:54:08,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T05:54:08,543 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:54:08,543 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:54:08,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:08,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:08,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:54:08,544 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T05:54:08,544 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1570076000, stopped=false 2024-11-16T05:54:08,545 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3456ee6a3164,40501,1731736439501 2024-11-16T05:54:08,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:54:08,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:54:08,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:54:08,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:08,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:08,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:08,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,547 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:54:08,547 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:54:08,547 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:54:08,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:08,547 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:54:08,547 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:54:08,548 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,42285,1731736440166' ***** 2024-11-16T05:54:08,548 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T05:54:08,548 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,37837,1731736440211' ***** 2024-11-16T05:54:08,548 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T05:54:08,548 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:54:08,548 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:54:08,548 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:54:08,548 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:54:08,549 INFO [RS:2;3456ee6a3164:37837 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:54:08,549 INFO [RS:1;3456ee6a3164:42285 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:54:08,549 INFO [RS:1;3456ee6a3164:42285 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:54:08,549 INFO [RS:2;3456ee6a3164:37837 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:54:08,549 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(3091): Received CLOSE for da031f93bd094d06fcd97a63a4675032 2024-11-16T05:54:08,549 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for df6f5bd741b507073cb45265746143fa 2024-11-16T05:54:08,549 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(3091): Received CLOSE for fa4cd934bc5e4b765d7bb157c1c600c0 2024-11-16T05:54:08,549 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for ac4bc85feea9c2820d87cf4b219cb17c 2024-11-16T05:54:08,549 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(3091): Received CLOSE for 8e3c5f7060b7491b30cd6cf5617a89c2 2024-11-16T05:54:08,549 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:08,549 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(3091): Received CLOSE for d63eb285e07e85739c793802fc7058c2 2024-11-16T05:54:08,549 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for bf20fbe8271ec5dfab659b24954f2df5 2024-11-16T05:54:08,549 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(3091): Received CLOSE for d426c5a1ddcc32067a58955e792d6dae 2024-11-16T05:54:08,549 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for 67d7f5593e349615c92abcb96869f4ba 2024-11-16T05:54:08,549 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(3091): Received CLOSE for f060fbfb898c418effc5c41d59bdb94f 2024-11-16T05:54:08,549 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for 1309086cc4268bb14438bcc29f5b63ff 2024-11-16T05:54:08,550 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(3091): Received CLOSE for fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for 6ba2670ce022e672b048093a692f3cb8 2024-11-16T05:54:08,550 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(3091): Received CLOSE for 7df86faabb66872de7c73da2806f6ac3 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for 83571bc94223ec9eaf85f80b241d7135 2024-11-16T05:54:08,550 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(3091): Received CLOSE for ceb8a4dcf270b48605e90af8f3b8eb2c 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for 3c48229c22c1b63769411754c30d2dfe 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for c342bb403770b75ff0a582854ab71fbe 2024-11-16T05:54:08,550 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for 20aedf13a99ac07f411f959c4c4bed3f 2024-11-16T05:54:08,550 INFO [RS:2;3456ee6a3164:37837 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for 83edddaeaac4b866615839a3e7d98099 2024-11-16T05:54:08,550 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing da031f93bd094d06fcd97a63a4675032, disabling compactions & flushes 2024-11-16T05:54:08,550 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df6f5bd741b507073cb45265746143fa, disabling compactions & flushes 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,550 INFO [RS:2;3456ee6a3164:37837 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;3456ee6a3164:37837. 2024-11-16T05:54:08,550 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:08,550 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:54:08,550 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:08,550 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:08,550 DEBUG [RS:2;3456ee6a3164:37837 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:54:08,550 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. after waiting 0 ms 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;3456ee6a3164:42285. 2024-11-16T05:54:08,550 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. after waiting 0 ms 2024-11-16T05:54:08,550 DEBUG [RS:2;3456ee6a3164:37837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:08,550 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:08,550 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:08,550 DEBUG [RS:1;3456ee6a3164:42285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:54:08,550 DEBUG [RS:1;3456ee6a3164:42285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:08,550 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:54:08,550 DEBUG [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(1325): Online Regions={da031f93bd094d06fcd97a63a4675032=test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032., fa4cd934bc5e4b765d7bb157c1c600c0=test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0., 8e3c5f7060b7491b30cd6cf5617a89c2=test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2., d63eb285e07e85739c793802fc7058c2=test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2., d426c5a1ddcc32067a58955e792d6dae=test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae., f060fbfb898c418effc5c41d59bdb94f=test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f., fdf8bdb7617bd8bb875a900a673e25fc=test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc., 7df86faabb66872de7c73da2806f6ac3=test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3., ceb8a4dcf270b48605e90af8f3b8eb2c=test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c.} 2024-11-16T05:54:08,550 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:54:08,551 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:54:08,551 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T05:54:08,551 DEBUG [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(1351): Waiting on 52d4c19e400f7867ccce41418f251b74, 7df86faabb66872de7c73da2806f6ac3, 8e3c5f7060b7491b30cd6cf5617a89c2, ceb8a4dcf270b48605e90af8f3b8eb2c, d426c5a1ddcc32067a58955e792d6dae, d63eb285e07e85739c793802fc7058c2, da031f93bd094d06fcd97a63a4675032, f060fbfb898c418effc5c41d59bdb94f, fa4cd934bc5e4b765d7bb157c1c600c0, fdf8bdb7617bd8bb875a900a673e25fc 2024-11-16T05:54:08,552 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-16T05:54:08,552 DEBUG [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(1351): Waiting on 1309086cc4268bb14438bcc29f5b63ff, 1588230740, 20aedf13a99ac07f411f959c4c4bed3f, 3c48229c22c1b63769411754c30d2dfe, 67d7f5593e349615c92abcb96869f4ba, 6ba2670ce022e672b048093a692f3cb8, 83571bc94223ec9eaf85f80b241d7135, 83edddaeaac4b866615839a3e7d98099, a62176f809b89002f2152e96ba4bef24, ac4bc85feea9c2820d87cf4b219cb17c, bf20fbe8271ec5dfab659b24954f2df5, c342bb403770b75ff0a582854ab71fbe, df6f5bd741b507073cb45265746143fa, e2536762964ceee99f3f786e0dad8403 2024-11-16T05:54:08,552 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:54:08,552 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:54:08,552 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:54:08,552 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:54:08,552 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:54:08,552 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-16T05:54:08,558 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/da031f93bd094d06fcd97a63a4675032/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,559 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:08,559 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for da031f93bd094d06fcd97a63a4675032: Waiting for close lock at 1731736448549Running coprocessor pre-close hooks at 1731736448550 (+1 ms)Disabling compacts and flushes for region at 1731736448550Disabling writes for close at 1731736448550Writing region close event to WAL at 1731736448552 (+2 ms)Running coprocessor post-close hooks at 1731736448558 (+6 ms)Closed at 1731736448558 2024-11-16T05:54:08,559 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731736442349.da031f93bd094d06fcd97a63a4675032. 2024-11-16T05:54:08,559 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fa4cd934bc5e4b765d7bb157c1c600c0, disabling compactions & flushes 2024-11-16T05:54:08,559 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:08,559 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:08,559 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. after waiting 0 ms 2024-11-16T05:54:08,559 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:08,566 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/df6f5bd741b507073cb45265746143fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,567 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:08,567 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df6f5bd741b507073cb45265746143fa: Waiting for close lock at 1731736448549Running coprocessor pre-close hooks at 1731736448550 (+1 ms)Disabling compacts and flushes for region at 1731736448550Disabling writes for close at 1731736448550Writing region close event to WAL at 1731736448551 (+1 ms)Running coprocessor post-close hooks at 1731736448567 (+16 ms)Closed at 1731736448567 2024-11-16T05:54:08,567 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731736442349.df6f5bd741b507073cb45265746143fa. 2024-11-16T05:54:08,567 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ac4bc85feea9c2820d87cf4b219cb17c, disabling compactions & flushes 2024-11-16T05:54:08,568 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:08,568 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:08,568 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. after waiting 0 ms 2024-11-16T05:54:08,568 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:08,568 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/info/362f04dfa2a44ada8973c46af461bc82 is 138, key is test,eee,1731736442349.6a7892b4caedb942fa23cc8a2f9d3b15./info:regioninfo/1731736447737/Put/seqid=0 2024-11-16T05:54:08,574 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/ac4bc85feea9c2820d87cf4b219cb17c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,575 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:08,575 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ac4bc85feea9c2820d87cf4b219cb17c: Waiting for close lock at 1731736448567Running coprocessor pre-close hooks at 1731736448567Disabling compacts and flushes for region at 1731736448567Disabling writes for close at 1731736448568 (+1 ms)Writing region close event to WAL at 1731736448568Running coprocessor post-close hooks at 1731736448575 (+7 ms)Closed at 1731736448575 2024-11-16T05:54:08,575 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731736442349.ac4bc85feea9c2820d87cf4b219cb17c. 2024-11-16T05:54:08,575 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e2536762964ceee99f3f786e0dad8403, disabling compactions & flushes 2024-11-16T05:54:08,575 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:08,575 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:08,575 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. after waiting 0 ms 2024-11-16T05:54:08,575 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:08,578 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/fa4cd934bc5e4b765d7bb157c1c600c0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,579 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:08,579 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fa4cd934bc5e4b765d7bb157c1c600c0: Waiting for close lock at 1731736448559Running coprocessor pre-close hooks at 1731736448559Disabling compacts and flushes for region at 1731736448559Disabling writes for close at 1731736448559Writing region close event to WAL at 1731736448569 (+10 ms)Running coprocessor post-close hooks at 1731736448578 (+9 ms)Closed at 1731736448578 2024-11-16T05:54:08,579 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731736442349.fa4cd934bc5e4b765d7bb157c1c600c0. 2024-11-16T05:54:08,579 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8e3c5f7060b7491b30cd6cf5617a89c2, disabling compactions & flushes 2024-11-16T05:54:08,579 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:08,579 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:08,579 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. after waiting 0 ms 2024-11-16T05:54:08,579 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:08,593 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/8e3c5f7060b7491b30cd6cf5617a89c2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,593 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/e2536762964ceee99f3f786e0dad8403/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,595 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:54:08,595 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:08,595 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:08,595 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e2536762964ceee99f3f786e0dad8403: Waiting for close lock at 1731736448575Running coprocessor pre-close hooks at 1731736448575Disabling compacts and flushes for region at 1731736448575Disabling writes for close at 1731736448575Writing region close event to WAL at 1731736448581 (+6 ms)Running coprocessor post-close hooks at 1731736448595 (+14 ms)Closed at 1731736448595 2024-11-16T05:54:08,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741869_1045 (size=8195) 2024-11-16T05:54:08,595 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731736442349.e2536762964ceee99f3f786e0dad8403. 2024-11-16T05:54:08,595 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8e3c5f7060b7491b30cd6cf5617a89c2: Waiting for close lock at 1731736448579Running coprocessor pre-close hooks at 1731736448579Disabling compacts and flushes for region at 1731736448579Disabling writes for close at 1731736448579Writing region close event to WAL at 1731736448585 (+6 ms)Running coprocessor post-close hooks at 1731736448595 (+10 ms)Closed at 1731736448595 2024-11-16T05:54:08,596 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731736442349.8e3c5f7060b7491b30cd6cf5617a89c2. 2024-11-16T05:54:08,596 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bf20fbe8271ec5dfab659b24954f2df5, disabling compactions & flushes 2024-11-16T05:54:08,596 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:08,596 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d63eb285e07e85739c793802fc7058c2, disabling compactions & flushes 2024-11-16T05:54:08,596 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:08,596 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:08,596 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:08,596 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:54:08,596 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. after waiting 0 ms 2024-11-16T05:54:08,596 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:08,596 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. after waiting 0 ms 2024-11-16T05:54:08,596 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:08,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741869_1045 (size=8195) 2024-11-16T05:54:08,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741869_1045 (size=8195) 2024-11-16T05:54:08,597 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/info/362f04dfa2a44ada8973c46af461bc82 2024-11-16T05:54:08,602 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d63eb285e07e85739c793802fc7058c2/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T05:54:08,602 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/bf20fbe8271ec5dfab659b24954f2df5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,603 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:08,603 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d63eb285e07e85739c793802fc7058c2: Waiting for close lock at 1731736448596Running coprocessor pre-close hooks at 1731736448596Disabling compacts and flushes for region at 1731736448596Disabling writes for close at 1731736448596Writing region close event to WAL at 1731736448597 (+1 ms)Running coprocessor post-close hooks at 1731736448603 (+6 ms)Closed at 1731736448603 2024-11-16T05:54:08,603 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731736442349.d63eb285e07e85739c793802fc7058c2. 2024-11-16T05:54:08,603 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:08,603 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d426c5a1ddcc32067a58955e792d6dae, disabling compactions & flushes 2024-11-16T05:54:08,603 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:08,603 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bf20fbe8271ec5dfab659b24954f2df5: Waiting for close lock at 1731736448596Running coprocessor pre-close hooks at 1731736448596Disabling compacts and flushes for region at 1731736448596Disabling writes for close at 1731736448596Writing region close event to WAL at 1731736448597 (+1 ms)Running coprocessor post-close hooks at 1731736448603 (+6 ms)Closed at 1731736448603 2024-11-16T05:54:08,604 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:08,604 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731736442349.bf20fbe8271ec5dfab659b24954f2df5. 2024-11-16T05:54:08,604 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. after waiting 0 ms 2024-11-16T05:54:08,604 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:08,604 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 67d7f5593e349615c92abcb96869f4ba, disabling compactions & flushes 2024-11-16T05:54:08,604 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:08,604 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:08,604 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. after waiting 0 ms 2024-11-16T05:54:08,604 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:08,608 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/d426c5a1ddcc32067a58955e792d6dae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,609 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/67d7f5593e349615c92abcb96869f4ba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,609 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:08,609 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d426c5a1ddcc32067a58955e792d6dae: Waiting for close lock at 1731736448603Running coprocessor pre-close hooks at 1731736448603Disabling compacts and flushes for region at 1731736448603Disabling writes for close at 1731736448604 (+1 ms)Writing region close event to WAL at 1731736448604Running coprocessor post-close hooks at 1731736448609 (+5 ms)Closed at 1731736448609 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731736442349.d426c5a1ddcc32067a58955e792d6dae. 2024-11-16T05:54:08,610 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 67d7f5593e349615c92abcb96869f4ba: Waiting for close lock at 1731736448604Running coprocessor pre-close hooks at 1731736448604Disabling compacts and flushes for region at 1731736448604Disabling writes for close at 1731736448604Writing region close event to WAL at 1731736448604Running coprocessor post-close hooks at 1731736448610 (+6 ms)Closed at 1731736448610 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f060fbfb898c418effc5c41d59bdb94f, disabling compactions & flushes 2024-11-16T05:54:08,610 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731736442349.67d7f5593e349615c92abcb96869f4ba. 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. after waiting 0 ms 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1309086cc4268bb14438bcc29f5b63ff, disabling compactions & flushes 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/.tmp/info/362f04dfa2a44ada8973c46af461bc82 as hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/info/362f04dfa2a44ada8973c46af461bc82 2024-11-16T05:54:08,610 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. after waiting 0 ms 2024-11-16T05:54:08,610 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:08,615 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/1309086cc4268bb14438bcc29f5b63ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,615 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/f060fbfb898c418effc5c41d59bdb94f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,616 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:08,616 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1309086cc4268bb14438bcc29f5b63ff: Waiting for close lock at 1731736448610Running coprocessor pre-close hooks at 1731736448610Disabling compacts and flushes for region at 1731736448610Disabling writes for close at 1731736448610Writing region close event to WAL at 1731736448611 (+1 ms)Running coprocessor post-close hooks at 1731736448616 (+5 ms)Closed at 1731736448616 2024-11-16T05:54:08,616 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:08,616 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f060fbfb898c418effc5c41d59bdb94f: Waiting for close lock at 1731736448610Running coprocessor pre-close hooks at 1731736448610Disabling compacts and flushes for region at 1731736448610Disabling writes for close at 1731736448610Writing region close event to WAL at 1731736448611 (+1 ms)Running coprocessor post-close hooks at 1731736448616 (+5 ms)Closed at 1731736448616 2024-11-16T05:54:08,617 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731736442349.1309086cc4268bb14438bcc29f5b63ff. 2024-11-16T05:54:08,617 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731736442349.f060fbfb898c418effc5c41d59bdb94f. 2024-11-16T05:54:08,617 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6ba2670ce022e672b048093a692f3cb8, disabling compactions & flushes 2024-11-16T05:54:08,617 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:08,617 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fdf8bdb7617bd8bb875a900a673e25fc, disabling compactions & flushes 2024-11-16T05:54:08,617 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:08,617 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:08,617 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. after waiting 0 ms 2024-11-16T05:54:08,617 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:08,617 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:08,617 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. after waiting 0 ms 2024-11-16T05:54:08,617 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:08,620 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/info/362f04dfa2a44ada8973c46af461bc82, entries=24, sequenceid=75, filesize=8.0 K 2024-11-16T05:54:08,622 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 69ms, sequenceid=75, compaction requested=false 2024-11-16T05:54:08,622 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/6ba2670ce022e672b048093a692f3cb8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,623 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/fdf8bdb7617bd8bb875a900a673e25fc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,623 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:08,623 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6ba2670ce022e672b048093a692f3cb8: Waiting for close lock at 1731736448617Running coprocessor pre-close hooks at 1731736448617Disabling compacts and flushes for region at 1731736448617Disabling writes for close at 1731736448617Writing region close event to WAL at 1731736448618 (+1 ms)Running coprocessor post-close hooks at 1731736448623 (+5 ms)Closed at 1731736448623 2024-11-16T05:54:08,623 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731736442349.6ba2670ce022e672b048093a692f3cb8. 2024-11-16T05:54:08,624 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 83571bc94223ec9eaf85f80b241d7135, disabling compactions & flushes 2024-11-16T05:54:08,624 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:08,624 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:08,624 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:08,624 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. after waiting 0 ms 2024-11-16T05:54:08,624 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:08,624 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fdf8bdb7617bd8bb875a900a673e25fc: Waiting for close lock at 1731736448617Running coprocessor pre-close hooks at 1731736448617Disabling compacts and flushes for region at 1731736448617Disabling writes for close at 1731736448617Writing region close event to WAL at 1731736448618 (+1 ms)Running coprocessor post-close hooks at 1731736448624 (+6 ms)Closed at 1731736448624 2024-11-16T05:54:08,624 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731736442349.fdf8bdb7617bd8bb875a900a673e25fc. 2024-11-16T05:54:08,624 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7df86faabb66872de7c73da2806f6ac3, disabling compactions & flushes 2024-11-16T05:54:08,624 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:08,624 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:08,624 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. after waiting 0 ms 2024-11-16T05:54:08,624 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:08,629 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-16T05:54:08,629 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/7df86faabb66872de7c73da2806f6ac3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,629 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/83571bc94223ec9eaf85f80b241d7135/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,630 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:08,630 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7df86faabb66872de7c73da2806f6ac3: Waiting for close lock at 1731736448624Running coprocessor pre-close hooks at 1731736448624Disabling compacts and flushes for region at 1731736448624Disabling writes for close at 1731736448624Writing region close event to WAL at 1731736448625 (+1 ms)Running coprocessor post-close hooks at 1731736448630 (+5 ms)Closed at 1731736448630 2024-11-16T05:54:08,630 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:54:08,630 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:08,630 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 83571bc94223ec9eaf85f80b241d7135: Waiting for close lock at 1731736448624Running coprocessor pre-close hooks at 1731736448624Disabling compacts and flushes for region at 1731736448624Disabling writes for close at 1731736448624Writing region close event to WAL at 1731736448624Running coprocessor post-close hooks at 1731736448630 (+6 ms)Closed at 1731736448630 2024-11-16T05:54:08,630 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731736442349.7df86faabb66872de7c73da2806f6ac3. 2024-11-16T05:54:08,630 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:54:08,630 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731736442349.83571bc94223ec9eaf85f80b241d7135. 2024-11-16T05:54:08,630 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736448552Running coprocessor pre-close hooks at 1731736448552Disabling compacts and flushes for region at 1731736448552Disabling writes for close at 1731736448552Obtaining lock to block concurrent updates at 1731736448552Preparing flush snapshotting stores in 1588230740 at 1731736448552Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731736448560 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731736448561 (+1 ms)Flushing 1588230740/info: creating writer at 1731736448561Flushing 1588230740/info: appending metadata at 1731736448567 (+6 ms)Flushing 1588230740/info: closing flushed file at 1731736448567Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2677e8e0: reopening flushed file at 1731736448609 (+42 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 69ms, sequenceid=75, compaction requested=false at 1731736448622 (+13 ms)Writing region close event to WAL at 1731736448625 (+3 ms)Running coprocessor post-close hooks at 1731736448630 (+5 ms)Closed at 1731736448630 2024-11-16T05:54:08,630 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T05:54:08,630 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ceb8a4dcf270b48605e90af8f3b8eb2c, disabling compactions & flushes 2024-11-16T05:54:08,630 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3c48229c22c1b63769411754c30d2dfe, disabling compactions & flushes 2024-11-16T05:54:08,630 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:08,631 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:08,631 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:08,631 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:08,631 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. after waiting 0 ms 2024-11-16T05:54:08,631 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. after waiting 0 ms 2024-11-16T05:54:08,631 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:08,631 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:08,635 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/ceb8a4dcf270b48605e90af8f3b8eb2c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,635 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/3c48229c22c1b63769411754c30d2dfe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,636 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:08,636 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ceb8a4dcf270b48605e90af8f3b8eb2c: Waiting for close lock at 1731736448630Running coprocessor pre-close hooks at 1731736448630Disabling compacts and flushes for region at 1731736448630Disabling writes for close at 1731736448631 (+1 ms)Writing region close event to WAL at 1731736448631Running coprocessor post-close hooks at 1731736448636 (+5 ms)Closed at 1731736448636 2024-11-16T05:54:08,636 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:08,636 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731736442349.ceb8a4dcf270b48605e90af8f3b8eb2c. 2024-11-16T05:54:08,636 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3c48229c22c1b63769411754c30d2dfe: Waiting for close lock at 1731736448630Running coprocessor pre-close hooks at 1731736448630Disabling compacts and flushes for region at 1731736448630Disabling writes for close at 1731736448631 (+1 ms)Writing region close event to WAL at 1731736448631Running coprocessor post-close hooks at 1731736448636 (+5 ms)Closed at 1731736448636 2024-11-16T05:54:08,636 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731736442349.3c48229c22c1b63769411754c30d2dfe. 2024-11-16T05:54:08,636 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c342bb403770b75ff0a582854ab71fbe, disabling compactions & flushes 2024-11-16T05:54:08,636 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:08,636 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:08,636 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. after waiting 0 ms 2024-11-16T05:54:08,637 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:08,641 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/c342bb403770b75ff0a582854ab71fbe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,642 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:08,642 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c342bb403770b75ff0a582854ab71fbe: Waiting for close lock at 1731736448636Running coprocessor pre-close hooks at 1731736448636Disabling compacts and flushes for region at 1731736448636Disabling writes for close at 1731736448637 (+1 ms)Writing region close event to WAL at 1731736448637Running coprocessor post-close hooks at 1731736448642 (+5 ms)Closed at 1731736448642 2024-11-16T05:54:08,642 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731736442349.c342bb403770b75ff0a582854ab71fbe. 2024-11-16T05:54:08,642 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 20aedf13a99ac07f411f959c4c4bed3f, disabling compactions & flushes 2024-11-16T05:54:08,642 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:08,643 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:08,643 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. after waiting 0 ms 2024-11-16T05:54:08,643 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:08,647 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/20aedf13a99ac07f411f959c4c4bed3f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,648 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:08,648 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 20aedf13a99ac07f411f959c4c4bed3f: Waiting for close lock at 1731736448642Running coprocessor pre-close hooks at 1731736448642Disabling compacts and flushes for region at 1731736448642Disabling writes for close at 1731736448643 (+1 ms)Writing region close event to WAL at 1731736448643Running coprocessor post-close hooks at 1731736448648 (+5 ms)Closed at 1731736448648 2024-11-16T05:54:08,648 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731736442349.20aedf13a99ac07f411f959c4c4bed3f. 2024-11-16T05:54:08,648 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 83edddaeaac4b866615839a3e7d98099, disabling compactions & flushes 2024-11-16T05:54:08,648 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:08,648 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:08,648 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. after waiting 0 ms 2024-11-16T05:54:08,648 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:08,653 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/data/default/test/83edddaeaac4b866615839a3e7d98099/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:08,654 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:08,654 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 83edddaeaac4b866615839a3e7d98099: Waiting for close lock at 1731736448648Running coprocessor pre-close hooks at 1731736448648Disabling compacts and flushes for region at 1731736448648Disabling writes for close at 1731736448648Writing region close event to WAL at 1731736448649 (+1 ms)Running coprocessor post-close hooks at 1731736448654 (+5 ms)Closed at 1731736448654 2024-11-16T05:54:08,654 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731736442349.83edddaeaac4b866615839a3e7d98099. 2024-11-16T05:54:08,751 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,37837,1731736440211; all regions closed. 2024-11-16T05:54:08,752 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,42285,1731736440166; all regions closed. 2024-11-16T05:54:08,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741833_1009 (size=5194) 2024-11-16T05:54:08,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741833_1009 (size=5194) 2024-11-16T05:54:08,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741833_1009 (size=5194) 2024-11-16T05:54:08,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741865_1041 (size=4204) 2024-11-16T05:54:08,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741865_1041 (size=4204) 2024-11-16T05:54:08,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741865_1041 (size=4204) 2024-11-16T05:54:08,759 DEBUG [RS:2;3456ee6a3164:37837 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/oldWALs 2024-11-16T05:54:08,760 INFO [RS:2;3456ee6a3164:37837 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C37837%2C1731736440211:(num 1731736441540) 2024-11-16T05:54:08,760 DEBUG [RS:2;3456ee6a3164:37837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:08,760 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:54:08,760 INFO [RS:2;3456ee6a3164:37837 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:54:08,760 INFO [RS:2;3456ee6a3164:37837 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T05:54:08,760 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:54:08,761 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:54:08,761 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:54:08,761 INFO [RS:2;3456ee6a3164:37837 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:54:08,761 INFO [RS:2;3456ee6a3164:37837 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:54:08,761 WARN [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:08,761 DEBUG [RS:1;3456ee6a3164:42285 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/oldWALs 2024-11-16T05:54:08,761 WARN [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74., aborting... java.io.IOException: Failed to report opened region to master: test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:08,761 INFO [RS:1;3456ee6a3164:42285 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C42285%2C1731736440166.meta:.meta(num 1731736446995) 2024-11-16T05:54:08,762 ERROR [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2440): ***** ABORTING region server 3456ee6a3164,37837,1731736440211: Failed to open region test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ppp,1731736442349.52d4c19e400f7867ccce41418f251b74. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:08,762 ERROR [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T05:54:08,762 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T05:54:08,763 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T05:54:08,763 INFO [RS:2;3456ee6a3164:37837 {}] executor.ExecutorService(108): Executor-9-RS_OPEN_REGION-regionserver/3456ee6a3164:0 had [Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #39 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #40 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-16T05:54:08,763 INFO [RS:2;3456ee6a3164:37837 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37837 2024-11-16T05:54:08,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741834_1010 (size=6795) 2024-11-16T05:54:08,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741834_1010 (size=6795) 2024-11-16T05:54:08,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37837-0x100471864220003, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,37837,1731736440211 2024-11-16T05:54:08,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:54:08,766 INFO [RS:2;3456ee6a3164:37837 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:54:08,766 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T05:54:08,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741834_1010 (size=6795) 2024-11-16T05:54:08,767 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T05:54:08,767 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,37837,1731736440211] 2024-11-16T05:54:08,768 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,37837,1731736440211 already deleted, retry=false 2024-11-16T05:54:08,768 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3456ee6a3164,37837,1731736440211 expired; onlineServers=1 2024-11-16T05:54:08,768 DEBUG [RS:1;3456ee6a3164:42285 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/oldWALs 2024-11-16T05:54:08,768 INFO [RS:1;3456ee6a3164:42285 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C42285%2C1731736440166:(num 1731736441542) 2024-11-16T05:54:08,768 DEBUG [RS:1;3456ee6a3164:42285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:08,768 INFO [RS:1;3456ee6a3164:42285 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:54:08,768 INFO [RS:1;3456ee6a3164:42285 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:54:08,769 INFO [RS:1;3456ee6a3164:42285 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T05:54:08,769 INFO [RS:1;3456ee6a3164:42285 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:54:08,769 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:54:08,769 INFO [RS:1;3456ee6a3164:42285 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42285 2024-11-16T05:54:08,769 WARN [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:08,770 WARN [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24., aborting... java.io.IOException: Failed to report opened region to master: test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:08,770 ERROR [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2440): ***** ABORTING region server 3456ee6a3164,42285,1731736440166: Failed to open region test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,sss,1731736442349.a62176f809b89002f2152e96ba4bef24. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:08,770 ERROR [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T05:54:08,770 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T05:54:08,771 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T05:54:08,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x100471864220002, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,42285,1731736440166 2024-11-16T05:54:08,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:54:08,771 INFO [RS:1;3456ee6a3164:42285 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:54:08,772 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,42285,1731736440166] 2024-11-16T05:54:08,773 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,42285,1731736440166 already deleted, retry=false 2024-11-16T05:54:08,773 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T05:54:08,773 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3456ee6a3164,42285,1731736440166 expired; onlineServers=0 2024-11-16T05:54:08,773 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3456ee6a3164,40501,1731736439501' ***** 2024-11-16T05:54:08,773 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T05:54:08,774 INFO [M:0;3456ee6a3164:40501 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:54:08,774 INFO [M:0;3456ee6a3164:40501 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:54:08,774 DEBUG [M:0;3456ee6a3164:40501 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T05:54:08,774 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T05:54:08,774 DEBUG [M:0;3456ee6a3164:40501 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T05:54:08,774 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736441141 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736441141,5,FailOnTimeoutGroup] 2024-11-16T05:54:08,774 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736441137 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736441137,5,FailOnTimeoutGroup] 2024-11-16T05:54:08,774 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T05:54:08,774 INFO [M:0;3456ee6a3164:40501 {}] hbase.ChoreService(370): Chore service for: master/3456ee6a3164:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T05:54:08,774 INFO [M:0;3456ee6a3164:40501 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:54:08,775 DEBUG [M:0;3456ee6a3164:40501 {}] master.HMaster(1795): Stopping service threads 2024-11-16T05:54:08,775 INFO [M:0;3456ee6a3164:40501 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T05:54:08,775 INFO [M:0;3456ee6a3164:40501 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:08,775 INFO [M:0;3456ee6a3164:40501 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:08,775 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T05:54:08,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T05:54:08,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:08,777 DEBUG [M:0;3456ee6a3164:40501 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-16T05:54:08,777 DEBUG [M:0;3456ee6a3164:40501 {}] master.ActiveMasterManager(353): master:40501-0x100471864220000, quorum=127.0.0.1:57132, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-16T05:54:08,777 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 529778944 }, "NonHeapMemoryUsage": { "committed": 164495360, "init": 7667712, "max": -1, "used": 161843656 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "3456ee6a3164", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 35779, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 104, "ProcessCallTime_min": 0, "ProcessCallTime_max": 16, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 8, "ProcessCallTime_75th_percentile": 12, "ProcessCallTime_90th_percentile": 14, "ProcessCallTime_95th_percentile": 15, "ProcessCallTime_98th_percentile": 15, "ProcessCallTime_99th_percentile": 15, "ProcessCallTime_99.9th_percentile": 15, "ProcessCallTime_TimeRangeCount_0-1": 104, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 104, "QueueCallTime_min": 0, "QueueCallTime_max": 10, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 7, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 9, "QueueCallTime_98th_percentile": 9, "QueueCallTime_99th_percentile": 9, "QueueCallTime_99.9th_percentile": 9, "QueueCallTime_TimeRangeCount_0-1": 104, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 104, "TotalCallTime_min": 0, "TotalCallTime_max": 16, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 8, "TotalCallTime_75th_percentile": 12, "TotalCallTime_90th_percentile": 14, "TotalCallTime_95th_percentile": 15, "TotalCallTime_98th_percentile": 15, "TotalCallTime_99th_percentile": 15, "TotalCallTime_99.9th_percentile": 15, "TotalCallTime_TimeRangeCount_0-1": 104, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 104, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 657, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 104, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 104, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 313, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 104, "sentBytes": 54773 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "3456ee6a3164", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57132", "tag.serverName": "3456ee6a3164,42285,1731736440166", "tag.clusterId": "5e221f76-f3d5-4494-a674-1fee78496748", "tag.Context": "regionserver", "tag.Hostname": "3456ee6a3164", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731736440166, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3341, "localBytesRead": 3341, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7474, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46703, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46703, "FlushOutputSize_max": 46703, "FlushOutputSize_mean": 46703, "FlushOutputSize_25th_percentile": 46703, "FlushOutputSize_median": 46703, "FlushOutputSize_75th_percentile": 46703, "FlushOutputSize_90th_percentile": 46703, "FlushOutputSize_95th_percentile": 46703, "FlushOutputSize_98th_percentile": 46703, "FlushOutputSize_99th_percentile": 46703, "FlushOutputSize_99.9th_percentile": 46703, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 272, "FlushTime_max": 272, "FlushTime_mean": 272, "FlushTime_25th_percentile": 272, "FlushTime_median": 272, "FlushTime_75th_percentile": 272, "FlushTime_90th_percentile": 272, "FlushTime_95th_percentile": 272, "FlushTime_98th_percentile": 272, "FlushTime_99th_percentile": 272, "FlushTime_99.9th_percentile": 272, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T05:54:08,778 INFO [M:0;3456ee6a3164:40501 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/.lastflushedseqids 2024-11-16T05:54:08,781 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 531876096 }, "NonHeapMemoryUsage": { "committed": 164560896, "init": 7667712, "max": -1, "used": 161928552 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "3456ee6a3164", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 35779, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 104, "ProcessCallTime_min": 0, "ProcessCallTime_max": 16, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 8, "ProcessCallTime_75th_percentile": 12, "ProcessCallTime_90th_percentile": 14, "ProcessCallTime_95th_percentile": 15, "ProcessCallTime_98th_percentile": 15, "ProcessCallTime_99th_percentile": 15, "ProcessCallTime_99.9th_percentile": 15, "ProcessCallTime_TimeRangeCount_0-1": 104, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 104, "QueueCallTime_min": 0, "QueueCallTime_max": 10, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 7, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 9, "QueueCallTime_98th_percentile": 9, "QueueCallTime_99th_percentile": 9, "QueueCallTime_99.9th_percentile": 9, "QueueCallTime_TimeRangeCount_0-1": 104, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 104, "TotalCallTime_min": 0, "TotalCallTime_max": 16, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 8, "TotalCallTime_75th_percentile": 12, "TotalCallTime_90th_percentile": 14, "TotalCallTime_95th_percentile": 15, "TotalCallTime_98th_percentile": 15, "TotalCallTime_99th_percentile": 15, "TotalCallTime_99.9th_percentile": 15, "TotalCallTime_TimeRangeCount_0-1": 104, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 104, "ResponseSize_min": 0, "ResponseSize_max": 22358, "ResponseSize_mean": 657, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16768, "ResponseSize_90th_percentile": 20122, "ResponseSize_95th_percentile": 21240, "ResponseSize_98th_percentile": 21910, "ResponseSize_99th_percentile": 22134, "ResponseSize_99.9th_percentile": 22335, "ResponseSize_SizeRangeCount_0-10": 104, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 104, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 313, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 104, "sentBytes": 54773 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "3456ee6a3164", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57132", "tag.serverName": "3456ee6a3164,42285,1731736440166", "tag.clusterId": "5e221f76-f3d5-4494-a674-1fee78496748", "tag.Context": "regionserver", "tag.Hostname": "3456ee6a3164", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731736440166, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3341, "localBytesRead": 3341, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7474, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46703, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46703, "FlushOutputSize_max": 46703, "FlushOutputSize_mean": 46703, "FlushOutputSize_25th_percentile": 46703, "FlushOutputSize_median": 46703, "FlushOutputSize_75th_percentile": 46703, "FlushOutputSize_90th_percentile": 46703, "FlushOutputSize_95th_percentile": 46703, "FlushOutputSize_98th_percentile": 46703, "FlushOutputSize_99th_percentile": 46703, "FlushOutputSize_99.9th_percentile": 46703, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 272, "FlushTime_max": 272, "FlushTime_mean": 272, "FlushTime_25th_percentile": 272, "FlushTime_median": 272, "FlushTime_75th_percentile": 272, "FlushTime_90th_percentile": 272, "FlushTime_95th_percentile": 272, "FlushTime_98th_percentile": 272, "FlushTime_99th_percentile": 272, "FlushTime_99.9th_percentile": 272, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-16T05:54:08,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741870_1046 (size=1) 2024-11-16T05:54:08,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741870_1046 (size=1) 2024-11-16T05:54:08,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741870_1046 (size=1) 2024-11-16T05:54:08,797 INFO [M:0;3456ee6a3164:40501 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:08,797 INFO [M:0;3456ee6a3164:40501 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T05:54:08,797 DEBUG [M:0;3456ee6a3164:40501 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:54:08,797 INFO [M:0;3456ee6a3164:40501 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:08,797 DEBUG [M:0;3456ee6a3164:40501 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:08,797 DEBUG [M:0;3456ee6a3164:40501 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:54:08,797 DEBUG [M:0;3456ee6a3164:40501 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:08,797 INFO [M:0;3456ee6a3164:40501 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.21 KB heapSize=315.05 KB 2024-11-16T05:54:08,814 DEBUG [M:0;3456ee6a3164:40501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d7fc555c31d414982e37d5e5012747a is 82, key is hbase:meta,,1/info:regioninfo/1731736447348/Put/seqid=0 2024-11-16T05:54:08,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46509 is added to blk_1073741871_1047 (size=6071) 2024-11-16T05:54:08,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34749 is added to blk_1073741871_1047 (size=6071) 2024-11-16T05:54:08,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39925 is added to blk_1073741871_1047 (size=6071) 2024-11-16T05:54:08,822 INFO [M:0;3456ee6a3164:40501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d7fc555c31d414982e37d5e5012747a 2024-11-16T05:54:08,848 DEBUG [M:0;3456ee6a3164:40501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43797/user/jenkins/test-data/c0833f85-ece8-2199-2ca1-8a235a774e79/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/596618a7bca74da9b4a54e90e0503630 is 2990, key is \x00\x00\x00\x00\x00\x00\x00