2024-12-07 07:13:22,932 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-07 07:13:22,948 main DEBUG Took 0.009531 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-07 07:13:22,948 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-07 07:13:22,949 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-07 07:13:22,950 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-07 07:13:22,951 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,957 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-07 07:13:22,969 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,971 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,972 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,973 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,973 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,974 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,975 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,976 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,976 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,977 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,978 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,978 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 07:13:22,979 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,979 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-07 07:13:22,980 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 07:13:22,981 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-07 07:13:22,983 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-07 07:13:22,983 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-07 07:13:22,984 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-07 07:13:22,985 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-07 07:13:22,992 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-07 07:13:22,994 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-07 07:13:22,996 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-07 07:13:22,996 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-07 07:13:22,997 main DEBUG createAppenders(={Console}) 2024-12-07 07:13:22,997 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-07 07:13:22,998 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-07 07:13:22,998 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-07 07:13:22,998 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-07 07:13:22,999 main DEBUG OutputStream closed 2024-12-07 07:13:22,999 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-07 07:13:22,999 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-07 07:13:22,999 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-07 07:13:23,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-07 07:13:23,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-07 07:13:23,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-07 07:13:23,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-07 07:13:23,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-07 07:13:23,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-07 07:13:23,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-07 07:13:23,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-07 07:13:23,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-07 07:13:23,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-07 07:13:23,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-07 07:13:23,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-07 07:13:23,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-07 07:13:23,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-07 07:13:23,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-07 07:13:23,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-07 07:13:23,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-07 07:13:23,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-07 07:13:23,076 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07 07:13:23,077 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-07 07:13:23,077 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-07 07:13:23,078 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-07T07:13:23,095 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-07 07:13:23,099 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-07 07:13:23,099 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07T07:13:23,319 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0 2024-12-07T07:13:23,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-07T07:13:23,352 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422, deleteOnExit=true 2024-12-07T07:13:23,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-07T07:13:23,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/test.cache.data in system properties and HBase conf 2024-12-07T07:13:23,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/hadoop.tmp.dir in system properties and HBase conf 2024-12-07T07:13:23,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/hadoop.log.dir in system properties and HBase conf 2024-12-07T07:13:23,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-07T07:13:23,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-07T07:13:23,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-07T07:13:23,450 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-07T07:13:23,534 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T07:13:23,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-07T07:13:23,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-07T07:13:23,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-07T07:13:23,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-07T07:13:23,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-07T07:13:23,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-07T07:13:23,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-07T07:13:23,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-07T07:13:23,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-07T07:13:23,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/nfs.dump.dir in system properties and HBase conf 2024-12-07T07:13:23,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/java.io.tmpdir in system properties and HBase conf 2024-12-07T07:13:23,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-07T07:13:23,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-07T07:13:23,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-07T07:13:24,530 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-07T07:13:24,591 INFO [Time-limited test {}] log.Log(170): Logging initialized @2253ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-07T07:13:24,652 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T07:13:24,707 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T07:13:24,727 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T07:13:24,727 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T07:13:24,728 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-07T07:13:24,740 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T07:13:24,742 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@952689a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/hadoop.log.dir/,AVAILABLE} 2024-12-07T07:13:24,743 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@378c34a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T07:13:24,931 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@302cfe51{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/java.io.tmpdir/jetty-localhost-35555-hadoop-hdfs-3_4_1-tests_jar-_-any-11119126660344810197/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-07T07:13:24,942 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c1eb5a9{HTTP/1.1, (http/1.1)}{localhost:35555} 2024-12-07T07:13:24,942 INFO [Time-limited test {}] server.Server(415): Started @2606ms 2024-12-07T07:13:25,512 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T07:13:25,518 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T07:13:25,518 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T07:13:25,519 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T07:13:25,519 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-07T07:13:25,520 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7de7e807{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/hadoop.log.dir/,AVAILABLE} 2024-12-07T07:13:25,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7edb5709{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T07:13:25,615 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@649cead0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/java.io.tmpdir/jetty-localhost-37953-hadoop-hdfs-3_4_1-tests_jar-_-any-14036531604706286862/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T07:13:25,616 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@742290fe{HTTP/1.1, (http/1.1)}{localhost:37953} 2024-12-07T07:13:25,616 INFO [Time-limited test {}] server.Server(415): Started @3279ms 2024-12-07T07:13:25,660 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T07:13:25,757 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T07:13:25,763 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T07:13:25,772 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T07:13:25,772 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T07:13:25,772 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-07T07:13:25,773 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e78dc0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/hadoop.log.dir/,AVAILABLE} 2024-12-07T07:13:25,774 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7913951{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T07:13:25,881 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@14ce6d63{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/java.io.tmpdir/jetty-localhost-41735-hadoop-hdfs-3_4_1-tests_jar-_-any-11630985913306052097/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T07:13:25,881 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@101b9d11{HTTP/1.1, (http/1.1)}{localhost:41735} 2024-12-07T07:13:25,882 INFO [Time-limited test {}] server.Server(415): Started @3545ms 2024-12-07T07:13:25,884 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T07:13:25,922 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T07:13:25,926 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T07:13:25,932 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T07:13:25,932 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T07:13:25,932 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-07T07:13:25,933 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76da50c3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/hadoop.log.dir/,AVAILABLE} 2024-12-07T07:13:25,933 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18213c95{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T07:13:26,050 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@78e5b4c2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/java.io.tmpdir/jetty-localhost-36905-hadoop-hdfs-3_4_1-tests_jar-_-any-13993443975555172743/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T07:13:26,051 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@687f641d{HTTP/1.1, (http/1.1)}{localhost:36905} 2024-12-07T07:13:26,051 INFO [Time-limited test {}] server.Server(415): Started @3714ms 2024-12-07T07:13:26,053 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T07:13:27,851 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data1/current/BP-125736803-172.17.0.2-1733555604035/current, will proceed with Du for space computation calculation, 2024-12-07T07:13:27,851 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data2/current/BP-125736803-172.17.0.2-1733555604035/current, will proceed with Du for space computation calculation, 2024-12-07T07:13:27,851 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data4/current/BP-125736803-172.17.0.2-1733555604035/current, will proceed with Du for space computation calculation, 2024-12-07T07:13:27,851 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data3/current/BP-125736803-172.17.0.2-1733555604035/current, will proceed with Du for space computation calculation, 2024-12-07T07:13:27,882 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T07:13:27,887 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T07:13:27,916 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data5/current/BP-125736803-172.17.0.2-1733555604035/current, will proceed with Du for space computation calculation, 2024-12-07T07:13:27,916 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data6/current/BP-125736803-172.17.0.2-1733555604035/current, will proceed with Du for space computation calculation, 2024-12-07T07:13:27,931 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x34b86dc17fef2726 with lease ID 0xf59416c23107b916: Processing first storage report for DS-0022ef86-b620-4ac8-b9f6-b3b559d4f8a4 from datanode DatanodeRegistration(127.0.0.1:44427, datanodeUuid=797ecaaa-a207-4210-908c-38d4a465da06, infoPort=42707, infoSecurePort=0, ipcPort=41673, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035) 2024-12-07T07:13:27,932 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x34b86dc17fef2726 with lease ID 0xf59416c23107b916: from storage DS-0022ef86-b620-4ac8-b9f6-b3b559d4f8a4 node DatanodeRegistration(127.0.0.1:44427, datanodeUuid=797ecaaa-a207-4210-908c-38d4a465da06, infoPort=42707, infoSecurePort=0, ipcPort=41673, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-07T07:13:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6043004794928675 with lease ID 0xf59416c23107b915: Processing first storage report for DS-db1aad08-c885-4b21-a48f-8bcb9ef25c82 from datanode DatanodeRegistration(127.0.0.1:41211, datanodeUuid=19740c13-1a4a-4982-a7f4-3a67004d3309, infoPort=37139, infoSecurePort=0, ipcPort=41933, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035) 2024-12-07T07:13:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6043004794928675 with lease ID 0xf59416c23107b915: from storage DS-db1aad08-c885-4b21-a48f-8bcb9ef25c82 node DatanodeRegistration(127.0.0.1:41211, datanodeUuid=19740c13-1a4a-4982-a7f4-3a67004d3309, infoPort=37139, infoSecurePort=0, ipcPort=41933, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-07T07:13:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x34b86dc17fef2726 with lease ID 0xf59416c23107b916: Processing first storage report for DS-a463d288-2707-418e-b418-4cad16b1455d from datanode DatanodeRegistration(127.0.0.1:44427, datanodeUuid=797ecaaa-a207-4210-908c-38d4a465da06, infoPort=42707, infoSecurePort=0, ipcPort=41673, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035) 2024-12-07T07:13:27,934 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x34b86dc17fef2726 with lease ID 0xf59416c23107b916: from storage DS-a463d288-2707-418e-b418-4cad16b1455d node DatanodeRegistration(127.0.0.1:44427, datanodeUuid=797ecaaa-a207-4210-908c-38d4a465da06, infoPort=42707, infoSecurePort=0, ipcPort=41673, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T07:13:27,934 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6043004794928675 with lease ID 0xf59416c23107b915: Processing first storage report for DS-8e490605-d3de-44f1-9e08-3ea9428419e6 from datanode DatanodeRegistration(127.0.0.1:41211, datanodeUuid=19740c13-1a4a-4982-a7f4-3a67004d3309, infoPort=37139, infoSecurePort=0, ipcPort=41933, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035) 2024-12-07T07:13:27,934 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6043004794928675 with lease ID 0xf59416c23107b915: from storage DS-8e490605-d3de-44f1-9e08-3ea9428419e6 node DatanodeRegistration(127.0.0.1:41211, datanodeUuid=19740c13-1a4a-4982-a7f4-3a67004d3309, infoPort=37139, infoSecurePort=0, ipcPort=41933, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-07T07:13:27,940 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T07:13:27,949 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2c68ad67b146c32e with lease ID 0xf59416c23107b917: Processing first storage report for DS-77296bb1-6178-4837-b9f0-b2f5cd04c8b3 from datanode DatanodeRegistration(127.0.0.1:38133, datanodeUuid=02bb7af9-b95d-49e2-ad84-77bc082e7c0c, infoPort=35525, infoSecurePort=0, ipcPort=45619, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035) 2024-12-07T07:13:27,949 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c68ad67b146c32e with lease ID 0xf59416c23107b917: from storage DS-77296bb1-6178-4837-b9f0-b2f5cd04c8b3 node DatanodeRegistration(127.0.0.1:38133, datanodeUuid=02bb7af9-b95d-49e2-ad84-77bc082e7c0c, infoPort=35525, infoSecurePort=0, ipcPort=45619, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T07:13:27,950 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2c68ad67b146c32e with lease ID 0xf59416c23107b917: Processing first storage report for DS-ee8167bc-8ee8-4e02-b648-095de760e532 from datanode DatanodeRegistration(127.0.0.1:38133, datanodeUuid=02bb7af9-b95d-49e2-ad84-77bc082e7c0c, infoPort=35525, infoSecurePort=0, ipcPort=45619, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035) 2024-12-07T07:13:27,950 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c68ad67b146c32e with lease ID 0xf59416c23107b917: from storage DS-ee8167bc-8ee8-4e02-b648-095de760e532 node DatanodeRegistration(127.0.0.1:38133, datanodeUuid=02bb7af9-b95d-49e2-ad84-77bc082e7c0c, infoPort=35525, infoSecurePort=0, ipcPort=45619, storageInfo=lv=-57;cid=testClusterID;nsid=1615932121;c=1733555604035), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T07:13:27,958 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0 2024-12-07T07:13:28,024 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/zookeeper_0, clientPort=63017, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T07:13:28,035 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63017 2024-12-07T07:13:28,045 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:28,048 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:28,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741825_1001 (size=7) 2024-12-07T07:13:28,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741825_1001 (size=7) 2024-12-07T07:13:28,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741825_1001 (size=7) 2024-12-07T07:13:28,688 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 with version=8 2024-12-07T07:13:28,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/hbase-staging 2024-12-07T07:13:28,766 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-07T07:13:29,047 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5d4d7471f19e:0 server-side Connection retries=45 2024-12-07T07:13:29,055 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,055 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,059 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T07:13:29,059 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,059 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T07:13:29,169 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T07:13:29,216 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-07T07:13:29,224 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-07T07:13:29,227 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T07:13:29,249 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2997 (auto-detected) 2024-12-07T07:13:29,249 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-07T07:13:29,267 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46437 2024-12-07T07:13:29,287 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46437 connecting to ZooKeeper ensemble=127.0.0.1:63017 2024-12-07T07:13:29,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:464370x0, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T07:13:29,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46437-0x101aea355730000 connected 2024-12-07T07:13:29,506 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:29,510 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:29,522 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T07:13:29,526 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658, hbase.cluster.distributed=false 2024-12-07T07:13:29,547 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T07:13:29,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46437 2024-12-07T07:13:29,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46437 2024-12-07T07:13:29,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46437 2024-12-07T07:13:29,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46437 2024-12-07T07:13:29,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46437 2024-12-07T07:13:29,650 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5d4d7471f19e:0 server-side Connection retries=45 2024-12-07T07:13:29,651 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,652 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,652 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T07:13:29,652 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,652 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T07:13:29,655 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T07:13:29,657 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T07:13:29,658 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34103 2024-12-07T07:13:29,660 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34103 connecting to ZooKeeper ensemble=127.0.0.1:63017 2024-12-07T07:13:29,661 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:29,664 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:29,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341030x0, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T07:13:29,679 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:341030x0, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T07:13:29,679 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34103-0x101aea355730001 connected 2024-12-07T07:13:29,682 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T07:13:29,689 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T07:13:29,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T07:13:29,696 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T07:13:29,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34103 2024-12-07T07:13:29,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34103 2024-12-07T07:13:29,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34103 2024-12-07T07:13:29,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34103 2024-12-07T07:13:29,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34103 2024-12-07T07:13:29,713 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5d4d7471f19e:0 server-side Connection retries=45 2024-12-07T07:13:29,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,714 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T07:13:29,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T07:13:29,715 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T07:13:29,715 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T07:13:29,716 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37405 2024-12-07T07:13:29,717 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37405 connecting to ZooKeeper ensemble=127.0.0.1:63017 2024-12-07T07:13:29,718 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:29,721 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:29,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374050x0, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T07:13:29,736 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:374050x0, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T07:13:29,736 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37405-0x101aea355730002 connected 2024-12-07T07:13:29,736 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T07:13:29,737 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T07:13:29,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T07:13:29,741 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T07:13:29,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37405 2024-12-07T07:13:29,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37405 2024-12-07T07:13:29,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37405 2024-12-07T07:13:29,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37405 2024-12-07T07:13:29,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37405 2024-12-07T07:13:29,760 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5d4d7471f19e:0 server-side Connection retries=45 2024-12-07T07:13:29,761 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,761 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,761 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T07:13:29,761 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T07:13:29,761 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T07:13:29,761 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T07:13:29,762 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T07:13:29,763 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41097 2024-12-07T07:13:29,765 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41097 connecting to ZooKeeper ensemble=127.0.0.1:63017 2024-12-07T07:13:29,766 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:29,769 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:29,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:410970x0, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T07:13:29,784 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:410970x0, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T07:13:29,784 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41097-0x101aea355730003 connected 2024-12-07T07:13:29,785 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T07:13:29,786 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T07:13:29,787 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T07:13:29,789 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T07:13:29,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41097 2024-12-07T07:13:29,790 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41097 2024-12-07T07:13:29,790 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41097 2024-12-07T07:13:29,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41097 2024-12-07T07:13:29,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41097 2024-12-07T07:13:29,813 DEBUG [M:0;5d4d7471f19e:46437 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5d4d7471f19e:46437 2024-12-07T07:13:29,814 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:29,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T07:13:29,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T07:13:29,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T07:13:29,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T07:13:29,832 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:29,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T07:13:29,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T07:13:29,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:29,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:29,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:29,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T07:13:29,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:29,862 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T07:13:29,864 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5d4d7471f19e,46437,1733555608905 from backup master directory 2024-12-07T07:13:29,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T07:13:29,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:29,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T07:13:29,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T07:13:29,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T07:13:29,879 WARN [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T07:13:29,879 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:29,882 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-07T07:13:29,884 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-07T07:13:29,937 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/hbase.id] with ID: f8b50c6c-024c-463b-b91f-59e96ac11d5c 2024-12-07T07:13:29,937 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/.tmp/hbase.id 2024-12-07T07:13:29,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741826_1002 (size=42) 2024-12-07T07:13:29,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741826_1002 (size=42) 2024-12-07T07:13:29,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741826_1002 (size=42) 2024-12-07T07:13:29,952 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/.tmp/hbase.id]:[hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/hbase.id] 2024-12-07T07:13:29,995 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T07:13:30,000 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T07:13:30,016 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-07T07:13:30,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741827_1003 (size=196) 2024-12-07T07:13:30,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741827_1003 (size=196) 2024-12-07T07:13:30,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741827_1003 (size=196) 2024-12-07T07:13:30,065 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T07:13:30,068 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T07:13:30,081 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T07:13:30,084 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T07:13:30,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741828_1004 (size=1189) 2024-12-07T07:13:30,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741828_1004 (size=1189) 2024-12-07T07:13:30,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741828_1004 (size=1189) 2024-12-07T07:13:30,133 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store 2024-12-07T07:13:30,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741829_1005 (size=34) 2024-12-07T07:13:30,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741829_1005 (size=34) 2024-12-07T07:13:30,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741829_1005 (size=34) 2024-12-07T07:13:30,158 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-07T07:13:30,162 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:30,164 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T07:13:30,164 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T07:13:30,164 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T07:13:30,165 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T07:13:30,166 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T07:13:30,166 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T07:13:30,167 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733555610163Disabling compacts and flushes for region at 1733555610163Disabling writes for close at 1733555610165 (+2 ms)Writing region close event to WAL at 1733555610166 (+1 ms)Closed at 1733555610166 2024-12-07T07:13:30,169 WARN [master/5d4d7471f19e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/.initializing 2024-12-07T07:13:30,169 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/WALs/5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:30,176 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T07:13:30,189 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5d4d7471f19e%2C46437%2C1733555608905, suffix=, logDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/WALs/5d4d7471f19e,46437,1733555608905, archiveDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/oldWALs, maxLogs=10 2024-12-07T07:13:30,209 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/WALs/5d4d7471f19e,46437,1733555608905/5d4d7471f19e%2C46437%2C1733555608905.1733555610193, exclude list is [], retry=0 2024-12-07T07:13:30,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38133,DS-77296bb1-6178-4837-b9f0-b2f5cd04c8b3,DISK] 2024-12-07T07:13:30,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41211,DS-db1aad08-c885-4b21-a48f-8bcb9ef25c82,DISK] 2024-12-07T07:13:30,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44427,DS-0022ef86-b620-4ac8-b9f6-b3b559d4f8a4,DISK] 2024-12-07T07:13:30,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-07T07:13:30,260 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/WALs/5d4d7471f19e,46437,1733555608905/5d4d7471f19e%2C46437%2C1733555608905.1733555610193 2024-12-07T07:13:30,262 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37139:37139),(127.0.0.1/127.0.0.1:42707:42707),(127.0.0.1/127.0.0.1:35525:35525)] 2024-12-07T07:13:30,262 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T07:13:30,263 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:30,266 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,267 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,300 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,329 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T07:13:30,333 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:30,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:30,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,339 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T07:13:30,340 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:30,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:30,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T07:13:30,344 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:30,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:30,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,348 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T07:13:30,348 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:30,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:30,350 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,353 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,355 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,360 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,361 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,364 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T07:13:30,368 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T07:13:30,373 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:30,374 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74531519, jitterRate=0.11060617864131927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T07:13:30,380 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733555610279Initializing all the Stores at 1733555610281 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733555610282 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555610282Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555610283 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555610283Cleaning up temporary data from old regions at 1733555610361 (+78 ms)Region opened successfully at 1733555610380 (+19 ms) 2024-12-07T07:13:30,381 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T07:13:30,412 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c68ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5d4d7471f19e/172.17.0.2:0 2024-12-07T07:13:30,439 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T07:13:30,448 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:30,449 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:30,451 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:30,452 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-07T07:13:30,456 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-07T07:13:30,457 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T07:13:30,483 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T07:13:30,493 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T07:13:30,545 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T07:13:30,549 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T07:13:30,552 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T07:13:30,562 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T07:13:30,564 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T07:13:30,567 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T07:13:30,576 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T07:13:30,578 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T07:13:30,587 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T07:13:30,609 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T07:13:30,618 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T07:13:30,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T07:13:30,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T07:13:30,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T07:13:30,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T07:13:30,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,634 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5d4d7471f19e,46437,1733555608905, sessionid=0x101aea355730000, setting cluster-up flag (Was=false) 2024-12-07T07:13:30,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,693 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T07:13:30,698 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:30,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:30,755 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T07:13:30,759 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:30,768 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T07:13:30,796 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(746): ClusterId : f8b50c6c-024c-463b-b91f-59e96ac11d5c 2024-12-07T07:13:30,796 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(746): ClusterId : f8b50c6c-024c-463b-b91f-59e96ac11d5c 2024-12-07T07:13:30,796 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(746): ClusterId : f8b50c6c-024c-463b-b91f-59e96ac11d5c 2024-12-07T07:13:30,799 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T07:13:30,799 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T07:13:30,799 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T07:13:30,810 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T07:13:30,810 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T07:13:30,810 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T07:13:30,810 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T07:13:30,810 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T07:13:30,810 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T07:13:30,820 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T07:13:30,820 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T07:13:30,820 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T07:13:30,821 DEBUG [RS:2;5d4d7471f19e:41097 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128a8a5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5d4d7471f19e/172.17.0.2:0 2024-12-07T07:13:30,821 DEBUG [RS:1;5d4d7471f19e:37405 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44eba067, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5d4d7471f19e/172.17.0.2:0 2024-12-07T07:13:30,821 DEBUG [RS:0;5d4d7471f19e:34103 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9fddc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5d4d7471f19e/172.17.0.2:0 2024-12-07T07:13:30,835 DEBUG [RS:1;5d4d7471f19e:37405 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;5d4d7471f19e:37405 2024-12-07T07:13:30,838 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T07:13:30,839 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T07:13:30,839 DEBUG [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T07:13:30,840 DEBUG [RS:0;5d4d7471f19e:34103 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5d4d7471f19e:34103 2024-12-07T07:13:30,840 DEBUG [RS:2;5d4d7471f19e:41097 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;5d4d7471f19e:41097 2024-12-07T07:13:30,840 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T07:13:30,840 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T07:13:30,840 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T07:13:30,840 DEBUG [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T07:13:30,840 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T07:13:30,841 DEBUG [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T07:13:30,842 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(2659): reportForDuty to master=5d4d7471f19e,46437,1733555608905 with port=37405, startcode=1733555609713 2024-12-07T07:13:30,842 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(2659): reportForDuty to master=5d4d7471f19e,46437,1733555608905 with port=34103, startcode=1733555609616 2024-12-07T07:13:30,842 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(2659): reportForDuty to master=5d4d7471f19e,46437,1733555608905 with port=41097, startcode=1733555609760 2024-12-07T07:13:30,843 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:30,853 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T07:13:30,854 DEBUG [RS:0;5d4d7471f19e:34103 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T07:13:30,854 DEBUG [RS:2;5d4d7471f19e:41097 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T07:13:30,854 DEBUG [RS:1;5d4d7471f19e:37405 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T07:13:30,859 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T07:13:30,864 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5d4d7471f19e,46437,1733555608905 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T07:13:30,873 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5d4d7471f19e:0, corePoolSize=5, maxPoolSize=5 2024-12-07T07:13:30,874 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5d4d7471f19e:0, corePoolSize=5, maxPoolSize=5 2024-12-07T07:13:30,874 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5d4d7471f19e:0, corePoolSize=5, maxPoolSize=5 2024-12-07T07:13:30,874 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5d4d7471f19e:0, corePoolSize=5, maxPoolSize=5 2024-12-07T07:13:30,874 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5d4d7471f19e:0, corePoolSize=10, maxPoolSize=10 2024-12-07T07:13:30,876 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:30,877 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5d4d7471f19e:0, corePoolSize=2, maxPoolSize=2 2024-12-07T07:13:30,877 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:30,878 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555640878 2024-12-07T07:13:30,880 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T07:13:30,881 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T07:13:30,884 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:30,884 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T07:13:30,886 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T07:13:30,886 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T07:13:30,886 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T07:13:30,886 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T07:13:30,890 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54083, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T07:13:30,890 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52105, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T07:13:30,890 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51625, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T07:13:30,888 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:30,892 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T07:13:30,893 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:30,893 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T07:13:30,894 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T07:13:30,894 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T07:13:30,896 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T07:13:30,897 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T07:13:30,899 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:30,900 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5d4d7471f19e:0:becomeActiveMaster-HFileCleaner.large.0-1733555610898,5,FailOnTimeoutGroup] 2024-12-07T07:13:30,903 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5d4d7471f19e:0:becomeActiveMaster-HFileCleaner.small.0-1733555610900,5,FailOnTimeoutGroup] 2024-12-07T07:13:30,903 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:30,903 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T07:13:30,905 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:30,905 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:30,907 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:30,910 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] master.ServerManager(517): Registering regionserver=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:30,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741831_1007 (size=1321) 2024-12-07T07:13:30,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741831_1007 (size=1321) 2024-12-07T07:13:30,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741831_1007 (size=1321) 2024-12-07T07:13:30,943 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T07:13:30,943 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:30,943 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] master.ServerManager(517): Registering regionserver=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:30,943 DEBUG [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:30,943 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:30,943 DEBUG [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43061 2024-12-07T07:13:30,944 DEBUG [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T07:13:30,948 DEBUG [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:30,948 DEBUG [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43061 2024-12-07T07:13:30,949 DEBUG [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T07:13:30,954 DEBUG [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-07T07:13:30,955 WARN [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-07T07:13:30,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T07:13:30,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741832_1008 (size=32) 2024-12-07T07:13:30,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741832_1008 (size=32) 2024-12-07T07:13:30,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741832_1008 (size=32) 2024-12-07T07:13:30,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:30,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T07:13:30,972 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T07:13:30,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:30,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:30,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T07:13:30,976 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T07:13:30,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:30,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:30,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T07:13:30,980 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T07:13:30,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:30,982 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:30,982 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T07:13:30,984 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T07:13:30,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:30,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:30,986 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T07:13:30,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740 2024-12-07T07:13:30,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740 2024-12-07T07:13:30,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T07:13:30,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T07:13:30,991 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T07:13:30,993 DEBUG [RS:1;5d4d7471f19e:37405 {}] zookeeper.ZKUtil(111): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:30,994 WARN [RS:1;5d4d7471f19e:37405 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T07:13:30,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T07:13:30,994 INFO [RS:1;5d4d7471f19e:37405 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T07:13:30,994 DEBUG [RS:0;5d4d7471f19e:34103 {}] zookeeper.ZKUtil(111): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:30,994 DEBUG [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:30,994 WARN [RS:0;5d4d7471f19e:34103 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T07:13:30,994 INFO [RS:0;5d4d7471f19e:34103 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T07:13:30,994 DEBUG [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:30,995 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5d4d7471f19e,37405,1733555609713] 2024-12-07T07:13:30,996 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5d4d7471f19e,34103,1733555609616] 2024-12-07T07:13:30,998 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:30,999 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65550166, jitterRate=-0.023226410150527954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T07:13:31,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733555610968Initializing all the Stores at 1733555610969 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733555610969Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733555610970 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555610970Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733555610970Cleaning up temporary data from old regions at 1733555610990 (+20 ms)Region opened successfully at 1733555611002 (+12 ms) 2024-12-07T07:13:31,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T07:13:31,002 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T07:13:31,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T07:13:31,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T07:13:31,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T07:13:31,004 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T07:13:31,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733555611002Disabling compacts and flushes for region at 1733555611002Disabling writes for close at 1733555611002Writing region close event to WAL at 1733555611003 (+1 ms)Closed at 1733555611004 (+1 ms) 2024-12-07T07:13:31,007 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:31,008 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T07:13:31,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T07:13:31,019 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T07:13:31,019 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T07:13:31,023 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:31,025 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T07:13:31,035 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T07:13:31,035 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T07:13:31,040 INFO [RS:0;5d4d7471f19e:34103 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T07:13:31,040 INFO [RS:1;5d4d7471f19e:37405 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T07:13:31,040 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,040 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,043 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T07:13:31,043 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T07:13:31,048 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T07:13:31,048 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T07:13:31,049 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,049 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,050 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0, corePoolSize=2, maxPoolSize=2 2024-12-07T07:13:31,050 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,050 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,051 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0, corePoolSize=2, maxPoolSize=2 2024-12-07T07:13:31,051 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,051 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,051 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,051 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,051 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,051 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,051 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,051 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5d4d7471f19e:0, corePoolSize=3, maxPoolSize=3 2024-12-07T07:13:31,051 DEBUG [RS:0;5d4d7471f19e:34103 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5d4d7471f19e:0, corePoolSize=3, maxPoolSize=3 2024-12-07T07:13:31,051 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,051 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,051 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,052 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5d4d7471f19e:0, corePoolSize=3, maxPoolSize=3 2024-12-07T07:13:31,052 DEBUG [RS:1;5d4d7471f19e:37405 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5d4d7471f19e:0, corePoolSize=3, maxPoolSize=3 2024-12-07T07:13:31,052 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,052 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,052 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,052 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,052 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,052 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,052 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,052 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,34103,1733555609616-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T07:13:31,052 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,053 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,053 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,053 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,37405,1733555609713-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T07:13:31,056 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(2659): reportForDuty to master=5d4d7471f19e,46437,1733555608905 with port=41097, startcode=1733555609760 2024-12-07T07:13:31,057 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:31,057 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] master.ServerManager(517): Registering regionserver=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:31,060 DEBUG [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:31,060 DEBUG [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43061 2024-12-07T07:13:31,060 DEBUG [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T07:13:31,071 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T07:13:31,071 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T07:13:31,072 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,37405,1733555609713-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,072 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,34103,1733555609616-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,073 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,073 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,073 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.Replication(171): 5d4d7471f19e,37405,1733555609713 started 2024-12-07T07:13:31,073 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.Replication(171): 5d4d7471f19e,34103,1733555609616 started 2024-12-07T07:13:31,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T07:13:31,083 DEBUG [RS:2;5d4d7471f19e:41097 {}] zookeeper.ZKUtil(111): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:31,083 WARN [RS:2;5d4d7471f19e:41097 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T07:13:31,083 INFO [RS:2;5d4d7471f19e:41097 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T07:13:31,083 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5d4d7471f19e,41097,1733555609760] 2024-12-07T07:13:31,083 DEBUG [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:31,089 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T07:13:31,090 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,090 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,090 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(1482): Serving as 5d4d7471f19e,37405,1733555609713, RpcServer on 5d4d7471f19e/172.17.0.2:37405, sessionid=0x101aea355730002 2024-12-07T07:13:31,090 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(1482): Serving as 5d4d7471f19e,34103,1733555609616, RpcServer on 5d4d7471f19e/172.17.0.2:34103, sessionid=0x101aea355730001 2024-12-07T07:13:31,091 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T07:13:31,091 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T07:13:31,091 DEBUG [RS:1;5d4d7471f19e:37405 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:31,091 DEBUG [RS:0;5d4d7471f19e:34103 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:31,091 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5d4d7471f19e,37405,1733555609713' 2024-12-07T07:13:31,091 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5d4d7471f19e,34103,1733555609616' 2024-12-07T07:13:31,092 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T07:13:31,092 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T07:13:31,092 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T07:13:31,092 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T07:13:31,093 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T07:13:31,093 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T07:13:31,093 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T07:13:31,093 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T07:13:31,093 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T07:13:31,093 DEBUG [RS:0;5d4d7471f19e:34103 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:31,093 DEBUG [RS:1;5d4d7471f19e:37405 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:31,093 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5d4d7471f19e,34103,1733555609616' 2024-12-07T07:13:31,093 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5d4d7471f19e,37405,1733555609713' 2024-12-07T07:13:31,093 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T07:13:31,093 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T07:13:31,094 INFO [RS:2;5d4d7471f19e:41097 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T07:13:31,094 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,094 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T07:13:31,094 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T07:13:31,094 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T07:13:31,094 DEBUG [RS:1;5d4d7471f19e:37405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T07:13:31,095 DEBUG [RS:0;5d4d7471f19e:34103 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T07:13:31,095 INFO [RS:1;5d4d7471f19e:37405 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T07:13:31,095 INFO [RS:0;5d4d7471f19e:34103 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T07:13:31,095 INFO [RS:1;5d4d7471f19e:37405 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T07:13:31,095 INFO [RS:0;5d4d7471f19e:34103 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T07:13:31,096 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T07:13:31,096 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,096 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,096 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,096 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,097 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,097 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,097 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0, corePoolSize=2, maxPoolSize=2 2024-12-07T07:13:31,097 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,097 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,097 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,097 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,097 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,098 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5d4d7471f19e:0, corePoolSize=1, maxPoolSize=1 2024-12-07T07:13:31,098 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5d4d7471f19e:0, corePoolSize=3, maxPoolSize=3 2024-12-07T07:13:31,098 DEBUG [RS:2;5d4d7471f19e:41097 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5d4d7471f19e:0, corePoolSize=3, maxPoolSize=3 2024-12-07T07:13:31,100 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,100 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,100 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,101 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,101 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,101 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,41097,1733555609760-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T07:13:31,115 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T07:13:31,115 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,41097,1733555609760-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,115 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,115 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.Replication(171): 5d4d7471f19e,41097,1733555609760 started 2024-12-07T07:13:31,129 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,129 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(1482): Serving as 5d4d7471f19e,41097,1733555609760, RpcServer on 5d4d7471f19e/172.17.0.2:41097, sessionid=0x101aea355730003 2024-12-07T07:13:31,129 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T07:13:31,130 DEBUG [RS:2;5d4d7471f19e:41097 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:31,130 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5d4d7471f19e,41097,1733555609760' 2024-12-07T07:13:31,130 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T07:13:31,131 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T07:13:31,131 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T07:13:31,131 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T07:13:31,131 DEBUG [RS:2;5d4d7471f19e:41097 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:31,132 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5d4d7471f19e,41097,1733555609760' 2024-12-07T07:13:31,132 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T07:13:31,132 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T07:13:31,133 DEBUG [RS:2;5d4d7471f19e:41097 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T07:13:31,133 INFO [RS:2;5d4d7471f19e:41097 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T07:13:31,133 INFO [RS:2;5d4d7471f19e:41097 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T07:13:31,176 WARN [5d4d7471f19e:46437 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T07:13:31,199 INFO [RS:1;5d4d7471f19e:37405 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T07:13:31,199 INFO [RS:0;5d4d7471f19e:34103 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T07:13:31,202 INFO [RS:0;5d4d7471f19e:34103 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5d4d7471f19e%2C34103%2C1733555609616, suffix=, logDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,34103,1733555609616, archiveDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/oldWALs, maxLogs=32 2024-12-07T07:13:31,202 INFO [RS:1;5d4d7471f19e:37405 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5d4d7471f19e%2C37405%2C1733555609713, suffix=, logDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,37405,1733555609713, archiveDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/oldWALs, maxLogs=32 2024-12-07T07:13:31,219 DEBUG [RS:1;5d4d7471f19e:37405 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,37405,1733555609713/5d4d7471f19e%2C37405%2C1733555609713.1733555611205, exclude list is [], retry=0 2024-12-07T07:13:31,219 DEBUG [RS:0;5d4d7471f19e:34103 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,34103,1733555609616/5d4d7471f19e%2C34103%2C1733555609616.1733555611205, exclude list is [], retry=0 2024-12-07T07:13:31,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44427,DS-0022ef86-b620-4ac8-b9f6-b3b559d4f8a4,DISK] 2024-12-07T07:13:31,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38133,DS-77296bb1-6178-4837-b9f0-b2f5cd04c8b3,DISK] 2024-12-07T07:13:31,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41211,DS-db1aad08-c885-4b21-a48f-8bcb9ef25c82,DISK] 2024-12-07T07:13:31,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38133,DS-77296bb1-6178-4837-b9f0-b2f5cd04c8b3,DISK] 2024-12-07T07:13:31,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41211,DS-db1aad08-c885-4b21-a48f-8bcb9ef25c82,DISK] 2024-12-07T07:13:31,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44427,DS-0022ef86-b620-4ac8-b9f6-b3b559d4f8a4,DISK] 2024-12-07T07:13:31,228 INFO [RS:1;5d4d7471f19e:37405 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,37405,1733555609713/5d4d7471f19e%2C37405%2C1733555609713.1733555611205 2024-12-07T07:13:31,228 INFO [RS:0;5d4d7471f19e:34103 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,34103,1733555609616/5d4d7471f19e%2C34103%2C1733555609616.1733555611205 2024-12-07T07:13:31,232 DEBUG [RS:0;5d4d7471f19e:34103 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35525:35525),(127.0.0.1/127.0.0.1:37139:37139),(127.0.0.1/127.0.0.1:42707:42707)] 2024-12-07T07:13:31,232 DEBUG [RS:1;5d4d7471f19e:37405 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42707:42707),(127.0.0.1/127.0.0.1:37139:37139),(127.0.0.1/127.0.0.1:35525:35525)] 2024-12-07T07:13:31,233 INFO [RS:2;5d4d7471f19e:41097 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T07:13:31,236 INFO [RS:2;5d4d7471f19e:41097 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5d4d7471f19e%2C41097%2C1733555609760, suffix=, logDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760, archiveDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/oldWALs, maxLogs=32 2024-12-07T07:13:31,252 DEBUG [RS:2;5d4d7471f19e:41097 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760/5d4d7471f19e%2C41097%2C1733555609760.1733555611238, exclude list is [], retry=0 2024-12-07T07:13:31,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38133,DS-77296bb1-6178-4837-b9f0-b2f5cd04c8b3,DISK] 2024-12-07T07:13:31,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41211,DS-db1aad08-c885-4b21-a48f-8bcb9ef25c82,DISK] 2024-12-07T07:13:31,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44427,DS-0022ef86-b620-4ac8-b9f6-b3b559d4f8a4,DISK] 2024-12-07T07:13:31,262 INFO [RS:2;5d4d7471f19e:41097 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760/5d4d7471f19e%2C41097%2C1733555609760.1733555611238 2024-12-07T07:13:31,262 DEBUG [RS:2;5d4d7471f19e:41097 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37139:37139),(127.0.0.1/127.0.0.1:35525:35525),(127.0.0.1/127.0.0.1:42707:42707)] 2024-12-07T07:13:31,429 DEBUG [5d4d7471f19e:46437 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-07T07:13:31,435 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(204): Hosts are {5d4d7471f19e=0} racks are {/default-rack=0} 2024-12-07T07:13:31,441 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T07:13:31,441 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T07:13:31,441 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-07T07:13:31,441 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T07:13:31,441 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T07:13:31,441 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-07T07:13:31,441 INFO [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T07:13:31,441 INFO [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T07:13:31,441 INFO [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-07T07:13:31,441 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T07:13:31,447 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:31,453 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,41097,1733555609760, state=OPENING 2024-12-07T07:13:31,514 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T07:13:31,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:31,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:31,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:31,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:31,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:31,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:31,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:31,528 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:31,531 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:31,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,41097,1733555609760}] 2024-12-07T07:13:31,707 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T07:13:31,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41165, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T07:13:31,719 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T07:13:31,720 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T07:13:31,720 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T07:13:31,724 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5d4d7471f19e%2C41097%2C1733555609760.meta, suffix=.meta, logDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760, archiveDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/oldWALs, maxLogs=32 2024-12-07T07:13:31,738 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, exclude list is [], retry=0 2024-12-07T07:13:31,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41211,DS-db1aad08-c885-4b21-a48f-8bcb9ef25c82,DISK] 2024-12-07T07:13:31,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38133,DS-77296bb1-6178-4837-b9f0-b2f5cd04c8b3,DISK] 2024-12-07T07:13:31,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44427,DS-0022ef86-b620-4ac8-b9f6-b3b559d4f8a4,DISK] 2024-12-07T07:13:31,745 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta 2024-12-07T07:13:31,746 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35525:35525),(127.0.0.1/127.0.0.1:37139:37139),(127.0.0.1/127.0.0.1:42707:42707)] 2024-12-07T07:13:31,746 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T07:13:31,748 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T07:13:31,751 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T07:13:31,756 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T07:13:31,761 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T07:13:31,762 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:31,762 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T07:13:31,762 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T07:13:31,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T07:13:31,767 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T07:13:31,767 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:31,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:31,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T07:13:31,770 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T07:13:31,770 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:31,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:31,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T07:13:31,773 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T07:13:31,773 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:31,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:31,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T07:13:31,775 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T07:13:31,775 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:31,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:31,776 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T07:13:31,778 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740 2024-12-07T07:13:31,781 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740 2024-12-07T07:13:31,783 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T07:13:31,783 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T07:13:31,784 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T07:13:31,786 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T07:13:31,788 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72168676, jitterRate=0.0753970742225647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T07:13:31,788 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T07:13:31,789 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733555611763Writing region info on filesystem at 1733555611763Initializing all the Stores at 1733555611765 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733555611765Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733555611765Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555611765Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733555611765Cleaning up temporary data from old regions at 1733555611783 (+18 ms)Running coprocessor post-open hooks at 1733555611788 (+5 ms)Region opened successfully at 1733555611789 (+1 ms) 2024-12-07T07:13:31,795 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733555611698 2024-12-07T07:13:31,806 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T07:13:31,806 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T07:13:31,808 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:31,811 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,41097,1733555609760, state=OPEN 2024-12-07T07:13:31,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:31,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:31,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:31,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:31,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:31,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:31,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:31,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:31,841 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:31,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T07:13:31,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,41097,1733555609760 in 308 msec 2024-12-07T07:13:31,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T07:13:31,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 835 msec 2024-12-07T07:13:31,857 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:31,857 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T07:13:31,881 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T07:13:31,882 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1] 2024-12-07T07:13:31,903 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T07:13:31,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45147, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T07:13:31,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1290 sec 2024-12-07T07:13:31,925 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733555611925, completionTime=-1 2024-12-07T07:13:31,928 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-07T07:13:31,928 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:31,952 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-07T07:13:31,953 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555671953 2024-12-07T07:13:31,953 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555731953 2024-12-07T07:13:31,953 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-07T07:13:31,954 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:31,961 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,46437,1733555608905-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,961 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,46437,1733555608905-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,962 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,46437,1733555608905-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,963 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5d4d7471f19e:46437, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,963 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,964 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:31,970 DEBUG [master/5d4d7471f19e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T07:13:31,990 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.111sec 2024-12-07T07:13:31,992 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T07:13:31,993 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T07:13:31,994 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T07:13:31,994 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T07:13:31,994 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T07:13:31,995 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,46437,1733555608905-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T07:13:31,996 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,46437,1733555608905-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T07:13:32,000 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T07:13:32,001 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T07:13:32,001 INFO [master/5d4d7471f19e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5d4d7471f19e,46437,1733555608905-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T07:13:32,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e0ffd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T07:13:32,020 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-07T07:13:32,020 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-07T07:13:32,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5d4d7471f19e,46437,-1 for getting cluster id 2024-12-07T07:13:32,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T07:13:32,033 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f8b50c6c-024c-463b-b91f-59e96ac11d5c' 2024-12-07T07:13:32,034 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T07:13:32,035 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f8b50c6c-024c-463b-b91f-59e96ac11d5c" 2024-12-07T07:13:32,035 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384b5846, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T07:13:32,035 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5d4d7471f19e,46437,-1] 2024-12-07T07:13:32,037 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T07:13:32,040 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T07:13:32,040 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T07:13:32,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a85cbf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T07:13:32,043 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T07:13:32,050 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1] 2024-12-07T07:13:32,050 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T07:13:32,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T07:13:32,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:32,091 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T07:13:32,096 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:32,099 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5325c401 2024-12-07T07:13:32,101 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T07:13:32,104 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T07:13:32,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T07:13:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:32,129 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T07:13:32,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-07T07:13:32,132 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:32,134 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T07:13:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T07:13:32,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741837_1013 (size=358) 2024-12-07T07:13:32,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741837_1013 (size=358) 2024-12-07T07:13:32,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741837_1013 (size=358) 2024-12-07T07:13:32,158 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ff1807b6b64d6c4ca2c6c6d7a2cafbd3, NAME => 'test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,159 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cf320858c98aa00078ada211b555e73e, NAME => 'test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,160 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 58e2442015eb81a1979c8a3c79c8b4a4, NAME => 'test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741838_1014 (size=45) 2024-12-07T07:13:32,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741838_1014 (size=45) 2024-12-07T07:13:32,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741838_1014 (size=45) 2024-12-07T07:13:32,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 58e2442015eb81a1979c8a3c79c8b4a4, disabling compactions & flushes 2024-12-07T07:13:32,186 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:32,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:32,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. after waiting 0 ms 2024-12-07T07:13:32,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:32,186 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:32,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 58e2442015eb81a1979c8a3c79c8b4a4: Waiting for close lock at 1733555612186Disabling compacts and flushes for region at 1733555612186Disabling writes for close at 1733555612186Writing region close event to WAL at 1733555612186Closed at 1733555612186 2024-12-07T07:13:32,187 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 764f719c1b7c5872a4cc2b9d21f2dc24, NAME => 'test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741839_1015 (size=45) 2024-12-07T07:13:32,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741839_1015 (size=45) 2024-12-07T07:13:32,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741839_1015 (size=45) 2024-12-07T07:13:32,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing cf320858c98aa00078ada211b555e73e, disabling compactions & flushes 2024-12-07T07:13:32,201 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:32,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:32,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. after waiting 0 ms 2024-12-07T07:13:32,201 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:32,202 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:32,202 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for cf320858c98aa00078ada211b555e73e: Waiting for close lock at 1733555612201Disabling compacts and flushes for region at 1733555612201Disabling writes for close at 1733555612201Writing region close event to WAL at 1733555612202 (+1 ms)Closed at 1733555612202 2024-12-07T07:13:32,203 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6aadffb1e0d6160805fffee563de94ca, NAME => 'test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741840_1016 (size=42) 2024-12-07T07:13:32,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741840_1016 (size=42) 2024-12-07T07:13:32,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741840_1016 (size=42) 2024-12-07T07:13:32,216 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,216 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ff1807b6b64d6c4ca2c6c6d7a2cafbd3, disabling compactions & flushes 2024-12-07T07:13:32,216 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:32,216 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:32,216 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. after waiting 0 ms 2024-12-07T07:13:32,216 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:32,216 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:32,216 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ff1807b6b64d6c4ca2c6c6d7a2cafbd3: Waiting for close lock at 1733555612216Disabling compacts and flushes for region at 1733555612216Disabling writes for close at 1733555612216Writing region close event to WAL at 1733555612216Closed at 1733555612216 2024-12-07T07:13:32,217 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => acd8e0c860fdbce58ce803c15220afb0, NAME => 'test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741841_1017 (size=45) 2024-12-07T07:13:32,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741841_1017 (size=45) 2024-12-07T07:13:32,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741841_1017 (size=45) 2024-12-07T07:13:32,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,232 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 764f719c1b7c5872a4cc2b9d21f2dc24, disabling compactions & flushes 2024-12-07T07:13:32,232 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:32,232 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:32,232 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. after waiting 0 ms 2024-12-07T07:13:32,232 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:32,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741842_1018 (size=45) 2024-12-07T07:13:32,232 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:32,232 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 764f719c1b7c5872a4cc2b9d21f2dc24: Waiting for close lock at 1733555612232Disabling compacts and flushes for region at 1733555612232Disabling writes for close at 1733555612232Writing region close event to WAL at 1733555612232Closed at 1733555612232 2024-12-07T07:13:32,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741842_1018 (size=45) 2024-12-07T07:13:32,233 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5101f15d587bc41cd49e68708e4a5ed2, NAME => 'test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741842_1018 (size=45) 2024-12-07T07:13:32,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6aadffb1e0d6160805fffee563de94ca, disabling compactions & flushes 2024-12-07T07:13:32,235 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:32,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:32,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. after waiting 0 ms 2024-12-07T07:13:32,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:32,235 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:32,235 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6aadffb1e0d6160805fffee563de94ca: Waiting for close lock at 1733555612235Disabling compacts and flushes for region at 1733555612235Disabling writes for close at 1733555612235Writing region close event to WAL at 1733555612235Closed at 1733555612235 2024-12-07T07:13:32,236 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => caa1b434450bd250c5e2792eafa50df0, NAME => 'test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T07:13:32,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741843_1019 (size=45) 2024-12-07T07:13:32,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741843_1019 (size=45) 2024-12-07T07:13:32,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741843_1019 (size=45) 2024-12-07T07:13:32,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing acd8e0c860fdbce58ce803c15220afb0, disabling compactions & flushes 2024-12-07T07:13:32,253 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:32,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:32,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. after waiting 0 ms 2024-12-07T07:13:32,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:32,253 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:32,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for acd8e0c860fdbce58ce803c15220afb0: Waiting for close lock at 1733555612253Disabling compacts and flushes for region at 1733555612253Disabling writes for close at 1733555612253Writing region close event to WAL at 1733555612253Closed at 1733555612253 2024-12-07T07:13:32,254 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2cfa28a210d48360878dfe1333fd8e26, NAME => 'test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741844_1020 (size=45) 2024-12-07T07:13:32,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741844_1020 (size=45) 2024-12-07T07:13:32,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741844_1020 (size=45) 2024-12-07T07:13:32,273 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741845_1021 (size=45) 2024-12-07T07:13:32,273 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing caa1b434450bd250c5e2792eafa50df0, disabling compactions & flushes 2024-12-07T07:13:32,273 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:32,273 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:32,274 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. after waiting 0 ms 2024-12-07T07:13:32,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741845_1021 (size=45) 2024-12-07T07:13:32,274 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:32,274 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:32,274 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for caa1b434450bd250c5e2792eafa50df0: Waiting for close lock at 1733555612273Disabling compacts and flushes for region at 1733555612273Disabling writes for close at 1733555612274 (+1 ms)Writing region close event to WAL at 1733555612274Closed at 1733555612274 2024-12-07T07:13:32,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741845_1021 (size=45) 2024-12-07T07:13:32,275 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 51e3bd49877ea2b244de43a10f4eaa37, NAME => 'test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,276 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,276 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5101f15d587bc41cd49e68708e4a5ed2, disabling compactions & flushes 2024-12-07T07:13:32,276 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:32,277 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:32,277 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. after waiting 0 ms 2024-12-07T07:13:32,277 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:32,277 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:32,277 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5101f15d587bc41cd49e68708e4a5ed2: Waiting for close lock at 1733555612276Disabling compacts and flushes for region at 1733555612276Disabling writes for close at 1733555612277 (+1 ms)Writing region close event to WAL at 1733555612277Closed at 1733555612277 2024-12-07T07:13:32,278 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a24f56f7916756473277b23c11f5ce84, NAME => 'test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741846_1022 (size=45) 2024-12-07T07:13:32,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741846_1022 (size=45) 2024-12-07T07:13:32,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741846_1022 (size=45) 2024-12-07T07:13:32,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2cfa28a210d48360878dfe1333fd8e26, disabling compactions & flushes 2024-12-07T07:13:32,286 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:32,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:32,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. after waiting 0 ms 2024-12-07T07:13:32,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:32,286 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:32,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2cfa28a210d48360878dfe1333fd8e26: Waiting for close lock at 1733555612286Disabling compacts and flushes for region at 1733555612286Disabling writes for close at 1733555612286Writing region close event to WAL at 1733555612286Closed at 1733555612286 2024-12-07T07:13:32,287 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe956aa50065b9ae6300ba6efb3d715d, NAME => 'test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741848_1024 (size=45) 2024-12-07T07:13:32,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741848_1024 (size=45) 2024-12-07T07:13:32,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741848_1024 (size=45) 2024-12-07T07:13:32,297 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a24f56f7916756473277b23c11f5ce84, disabling compactions & flushes 2024-12-07T07:13:32,298 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:32,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:32,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. after waiting 0 ms 2024-12-07T07:13:32,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:32,298 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:32,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a24f56f7916756473277b23c11f5ce84: Waiting for close lock at 1733555612298Disabling compacts and flushes for region at 1733555612298Disabling writes for close at 1733555612298Writing region close event to WAL at 1733555612298Closed at 1733555612298 2024-12-07T07:13:32,299 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4c8344bda7e0c92708aba9a317b54627, NAME => 'test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741847_1023 (size=45) 2024-12-07T07:13:32,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741847_1023 (size=45) 2024-12-07T07:13:32,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741847_1023 (size=45) 2024-12-07T07:13:32,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 51e3bd49877ea2b244de43a10f4eaa37, disabling compactions & flushes 2024-12-07T07:13:32,313 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:32,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:32,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. after waiting 0 ms 2024-12-07T07:13:32,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:32,313 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:32,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 51e3bd49877ea2b244de43a10f4eaa37: Waiting for close lock at 1733555612313Disabling compacts and flushes for region at 1733555612313Disabling writes for close at 1733555612313Writing region close event to WAL at 1733555612313Closed at 1733555612313 2024-12-07T07:13:32,314 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 45e7ad8954e0a4d51f932a71d965426b, NAME => 'test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741849_1025 (size=45) 2024-12-07T07:13:32,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741850_1026 (size=45) 2024-12-07T07:13:32,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741849_1025 (size=45) 2024-12-07T07:13:32,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741850_1026 (size=45) 2024-12-07T07:13:32,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741850_1026 (size=45) 2024-12-07T07:13:32,318 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741849_1025 (size=45) 2024-12-07T07:13:32,318 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing fe956aa50065b9ae6300ba6efb3d715d, disabling compactions & flushes 2024-12-07T07:13:32,318 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:32,318 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:32,318 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. after waiting 0 ms 2024-12-07T07:13:32,318 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:32,318 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,318 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:32,318 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe956aa50065b9ae6300ba6efb3d715d: Waiting for close lock at 1733555612318Disabling compacts and flushes for region at 1733555612318Disabling writes for close at 1733555612318Writing region close event to WAL at 1733555612318Closed at 1733555612318 2024-12-07T07:13:32,318 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4c8344bda7e0c92708aba9a317b54627, disabling compactions & flushes 2024-12-07T07:13:32,318 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:32,319 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:32,319 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. after waiting 0 ms 2024-12-07T07:13:32,319 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c1e55fd1b7105eca2dcbd1d5897f7f5a, NAME => 'test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,319 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:32,319 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:32,319 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4c8344bda7e0c92708aba9a317b54627: Waiting for close lock at 1733555612318Disabling compacts and flushes for region at 1733555612318Disabling writes for close at 1733555612319 (+1 ms)Writing region close event to WAL at 1733555612319Closed at 1733555612319 2024-12-07T07:13:32,319 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c709bd9f88fc6dff6480583f422247fa, NAME => 'test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741851_1027 (size=45) 2024-12-07T07:13:32,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741851_1027 (size=45) 2024-12-07T07:13:32,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741851_1027 (size=45) 2024-12-07T07:13:32,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741852_1028 (size=45) 2024-12-07T07:13:32,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741852_1028 (size=45) 2024-12-07T07:13:32,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741852_1028 (size=45) 2024-12-07T07:13:32,332 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,332 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 45e7ad8954e0a4d51f932a71d965426b, disabling compactions & flushes 2024-12-07T07:13:32,332 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:32,332 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:32,332 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. after waiting 0 ms 2024-12-07T07:13:32,332 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:32,332 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:32,332 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 45e7ad8954e0a4d51f932a71d965426b: Waiting for close lock at 1733555612332Disabling compacts and flushes for region at 1733555612332Disabling writes for close at 1733555612332Writing region close event to WAL at 1733555612332Closed at 1733555612332 2024-12-07T07:13:32,333 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 37e2dfda6af2ae02ec7c14d6781d4503, NAME => 'test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,333 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c1e55fd1b7105eca2dcbd1d5897f7f5a, disabling compactions & flushes 2024-12-07T07:13:32,334 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:32,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:32,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. after waiting 0 ms 2024-12-07T07:13:32,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:32,334 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:32,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c1e55fd1b7105eca2dcbd1d5897f7f5a: Waiting for close lock at 1733555612334Disabling compacts and flushes for region at 1733555612334Disabling writes for close at 1733555612334Writing region close event to WAL at 1733555612334Closed at 1733555612334 2024-12-07T07:13:32,335 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ffbaf4a5440811c93c49fb27af711a24, NAME => 'test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741853_1029 (size=45) 2024-12-07T07:13:32,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741853_1029 (size=45) 2024-12-07T07:13:32,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741853_1029 (size=45) 2024-12-07T07:13:32,338 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,338 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c709bd9f88fc6dff6480583f422247fa, disabling compactions & flushes 2024-12-07T07:13:32,338 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:32,338 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:32,338 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. after waiting 0 ms 2024-12-07T07:13:32,338 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:32,339 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:32,339 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c709bd9f88fc6dff6480583f422247fa: Waiting for close lock at 1733555612338Disabling compacts and flushes for region at 1733555612338Disabling writes for close at 1733555612338Writing region close event to WAL at 1733555612339 (+1 ms)Closed at 1733555612339 2024-12-07T07:13:32,339 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ab678da1dd813ac72686a84c4f22fdc0, NAME => 'test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741854_1030 (size=45) 2024-12-07T07:13:32,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741854_1030 (size=45) 2024-12-07T07:13:32,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741854_1030 (size=45) 2024-12-07T07:13:32,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741855_1031 (size=45) 2024-12-07T07:13:32,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741855_1031 (size=45) 2024-12-07T07:13:32,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741855_1031 (size=45) 2024-12-07T07:13:32,353 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,353 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 37e2dfda6af2ae02ec7c14d6781d4503, disabling compactions & flushes 2024-12-07T07:13:32,353 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:32,353 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:32,353 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. after waiting 0 ms 2024-12-07T07:13:32,353 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:32,353 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:32,354 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 37e2dfda6af2ae02ec7c14d6781d4503: Waiting for close lock at 1733555612353Disabling compacts and flushes for region at 1733555612353Disabling writes for close at 1733555612353Writing region close event to WAL at 1733555612353Closed at 1733555612353 2024-12-07T07:13:32,354 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => da5554a80a6c43e6ba1b1014d9f4023a, NAME => 'test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741856_1032 (size=45) 2024-12-07T07:13:32,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741856_1032 (size=45) 2024-12-07T07:13:32,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741856_1032 (size=45) 2024-12-07T07:13:32,357 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,357 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ab678da1dd813ac72686a84c4f22fdc0, disabling compactions & flushes 2024-12-07T07:13:32,357 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:32,357 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:32,357 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. after waiting 0 ms 2024-12-07T07:13:32,358 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:32,358 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:32,358 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ab678da1dd813ac72686a84c4f22fdc0: Waiting for close lock at 1733555612357Disabling compacts and flushes for region at 1733555612357Disabling writes for close at 1733555612358 (+1 ms)Writing region close event to WAL at 1733555612358Closed at 1733555612358 2024-12-07T07:13:32,359 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5369f3b9703dae7de70d6556989c74a8, NAME => 'test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741857_1033 (size=45) 2024-12-07T07:13:32,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741857_1033 (size=45) 2024-12-07T07:13:32,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741857_1033 (size=45) 2024-12-07T07:13:32,367 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,367 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing da5554a80a6c43e6ba1b1014d9f4023a, disabling compactions & flushes 2024-12-07T07:13:32,368 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:32,368 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:32,368 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. after waiting 0 ms 2024-12-07T07:13:32,368 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:32,368 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:32,368 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for da5554a80a6c43e6ba1b1014d9f4023a: Waiting for close lock at 1733555612367Disabling compacts and flushes for region at 1733555612367Disabling writes for close at 1733555612368 (+1 ms)Writing region close event to WAL at 1733555612368Closed at 1733555612368 2024-12-07T07:13:32,369 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9d2616a94cd0a6eca969a2466add4db9, NAME => 'test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741858_1034 (size=45) 2024-12-07T07:13:32,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741858_1034 (size=45) 2024-12-07T07:13:32,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741858_1034 (size=45) 2024-12-07T07:13:32,374 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,375 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5369f3b9703dae7de70d6556989c74a8, disabling compactions & flushes 2024-12-07T07:13:32,375 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:32,375 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:32,375 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. after waiting 0 ms 2024-12-07T07:13:32,375 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:32,375 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:32,375 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5369f3b9703dae7de70d6556989c74a8: Waiting for close lock at 1733555612374Disabling compacts and flushes for region at 1733555612374Disabling writes for close at 1733555612375 (+1 ms)Writing region close event to WAL at 1733555612375Closed at 1733555612375 2024-12-07T07:13:32,376 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 99de37e290b4841263ba62805b625d97, NAME => 'test,www,1733555612106.99de37e290b4841263ba62805b625d97.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741859_1035 (size=45) 2024-12-07T07:13:32,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741859_1035 (size=45) 2024-12-07T07:13:32,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741859_1035 (size=45) 2024-12-07T07:13:32,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9d2616a94cd0a6eca969a2466add4db9, disabling compactions & flushes 2024-12-07T07:13:32,381 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:32,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:32,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. after waiting 0 ms 2024-12-07T07:13:32,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:32,381 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:32,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9d2616a94cd0a6eca969a2466add4db9: Waiting for close lock at 1733555612381Disabling compacts and flushes for region at 1733555612381Disabling writes for close at 1733555612381Writing region close event to WAL at 1733555612381Closed at 1733555612381 2024-12-07T07:13:32,382 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d60e3c4175109e97854d2b36c43a459f, NAME => 'test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741860_1036 (size=45) 2024-12-07T07:13:32,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741860_1036 (size=45) 2024-12-07T07:13:32,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741860_1036 (size=45) 2024-12-07T07:13:32,388 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1733555612106.99de37e290b4841263ba62805b625d97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,388 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 99de37e290b4841263ba62805b625d97, disabling compactions & flushes 2024-12-07T07:13:32,388 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:32,388 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:32,388 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733555612106.99de37e290b4841263ba62805b625d97. after waiting 0 ms 2024-12-07T07:13:32,388 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:32,388 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:32,388 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 99de37e290b4841263ba62805b625d97: Waiting for close lock at 1733555612388Disabling compacts and flushes for region at 1733555612388Disabling writes for close at 1733555612388Writing region close event to WAL at 1733555612388Closed at 1733555612388 2024-12-07T07:13:32,389 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d07cc4fd0d69fc55b90b071c1b0400c7, NAME => 'test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741861_1037 (size=45) 2024-12-07T07:13:32,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741861_1037 (size=45) 2024-12-07T07:13:32,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741861_1037 (size=45) 2024-12-07T07:13:32,393 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,394 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d60e3c4175109e97854d2b36c43a459f, disabling compactions & flushes 2024-12-07T07:13:32,394 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:32,394 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:32,394 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. after waiting 0 ms 2024-12-07T07:13:32,394 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:32,394 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:32,394 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d60e3c4175109e97854d2b36c43a459f: Waiting for close lock at 1733555612394Disabling compacts and flushes for region at 1733555612394Disabling writes for close at 1733555612394Writing region close event to WAL at 1733555612394Closed at 1733555612394 2024-12-07T07:13:32,395 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => abe9b4a81ffb7c674120d136c6ab6231, NAME => 'test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658 2024-12-07T07:13:32,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741862_1038 (size=45) 2024-12-07T07:13:32,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741862_1038 (size=45) 2024-12-07T07:13:32,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741862_1038 (size=45) 2024-12-07T07:13:32,412 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,412 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d07cc4fd0d69fc55b90b071c1b0400c7, disabling compactions & flushes 2024-12-07T07:13:32,412 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:32,412 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:32,412 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. after waiting 0 ms 2024-12-07T07:13:32,412 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:32,412 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:32,412 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d07cc4fd0d69fc55b90b071c1b0400c7: Waiting for close lock at 1733555612412Disabling compacts and flushes for region at 1733555612412Disabling writes for close at 1733555612412Writing region close event to WAL at 1733555612412Closed at 1733555612412 2024-12-07T07:13:32,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741863_1039 (size=42) 2024-12-07T07:13:32,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741863_1039 (size=42) 2024-12-07T07:13:32,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741863_1039 (size=42) 2024-12-07T07:13:32,420 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,420 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing abe9b4a81ffb7c674120d136c6ab6231, disabling compactions & flushes 2024-12-07T07:13:32,420 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:32,420 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:32,420 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. after waiting 0 ms 2024-12-07T07:13:32,420 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:32,420 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:32,420 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for abe9b4a81ffb7c674120d136c6ab6231: Waiting for close lock at 1733555612420Disabling compacts and flushes for region at 1733555612420Disabling writes for close at 1733555612420Writing region close event to WAL at 1733555612420Closed at 1733555612420 2024-12-07T07:13:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T07:13:32,754 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:32,755 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ffbaf4a5440811c93c49fb27af711a24, disabling compactions & flushes 2024-12-07T07:13:32,755 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:32,755 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:32,755 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. after waiting 0 ms 2024-12-07T07:13:32,755 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:32,756 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:32,756 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ffbaf4a5440811c93c49fb27af711a24: Waiting for close lock at 1733555612755Disabling compacts and flushes for region at 1733555612755Disabling writes for close at 1733555612755Writing region close event to WAL at 1733555612755Closed at 1733555612755 2024-12-07T07:13:32,759 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T07:13:32,766 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,766 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,766 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T07:13:32,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733555612106.99de37e290b4841263ba62805b625d97.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733555612760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733555612760"}]},"ts":"1733555612760"} 2024-12-07T07:13:32,805 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-07T07:13:32,807 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T07:13:32,810 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733555612807"}]},"ts":"1733555612807"} 2024-12-07T07:13:32,814 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-07T07:13:32,815 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {5d4d7471f19e=0} racks are {/default-rack=0} 2024-12-07T07:13:32,833 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T07:13:32,833 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T07:13:32,833 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-07T07:13:32,833 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T07:13:32,833 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T07:13:32,833 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-07T07:13:32,833 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T07:13:32,833 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T07:13:32,833 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-07T07:13:32,833 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T07:13:32,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=58e2442015eb81a1979c8a3c79c8b4a4, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cf320858c98aa00078ada211b555e73e, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=764f719c1b7c5872a4cc2b9d21f2dc24, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6aadffb1e0d6160805fffee563de94ca, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5101f15d587bc41cd49e68708e4a5ed2, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=caa1b434450bd250c5e2792eafa50df0, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51e3bd49877ea2b244de43a10f4eaa37, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a24f56f7916756473277b23c11f5ce84, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c8344bda7e0c92708aba9a317b54627, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=45e7ad8954e0a4d51f932a71d965426b, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c709bd9f88fc6dff6480583f422247fa, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=37e2dfda6af2ae02ec7c14d6781d4503, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ffbaf4a5440811c93c49fb27af711a24, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=da5554a80a6c43e6ba1b1014d9f4023a, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5369f3b9703dae7de70d6556989c74a8, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=99de37e290b4841263ba62805b625d97, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d60e3c4175109e97854d2b36c43a459f, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abe9b4a81ffb7c674120d136c6ab6231, ASSIGN}] 2024-12-07T07:13:32,844 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abe9b4a81ffb7c674120d136c6ab6231, ASSIGN 2024-12-07T07:13:32,846 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=abe9b4a81ffb7c674120d136c6ab6231, ASSIGN; state=OFFLINE, location=5d4d7471f19e,37405,1733555609713; forceNewPlan=false, retain=false 2024-12-07T07:13:32,847 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:32,848 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN; state=OFFLINE, location=5d4d7471f19e,41097,1733555609760; forceNewPlan=false, retain=false 2024-12-07T07:13:32,850 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d60e3c4175109e97854d2b36c43a459f, ASSIGN 2024-12-07T07:13:32,851 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d60e3c4175109e97854d2b36c43a459f, ASSIGN; state=OFFLINE, location=5d4d7471f19e,34103,1733555609616; forceNewPlan=false, retain=false 2024-12-07T07:13:32,852 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=99de37e290b4841263ba62805b625d97, ASSIGN 2024-12-07T07:13:32,854 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=99de37e290b4841263ba62805b625d97, ASSIGN; state=OFFLINE, location=5d4d7471f19e,37405,1733555609713; forceNewPlan=false, retain=false 2024-12-07T07:13:32,855 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:32,856 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN; state=OFFLINE, location=5d4d7471f19e,41097,1733555609760; forceNewPlan=false, retain=false 2024-12-07T07:13:32,858 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5369f3b9703dae7de70d6556989c74a8, ASSIGN 2024-12-07T07:13:32,859 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5369f3b9703dae7de70d6556989c74a8, ASSIGN; state=OFFLINE, location=5d4d7471f19e,34103,1733555609616; forceNewPlan=false, retain=false 2024-12-07T07:13:32,861 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=da5554a80a6c43e6ba1b1014d9f4023a, ASSIGN 2024-12-07T07:13:32,862 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=da5554a80a6c43e6ba1b1014d9f4023a, ASSIGN; state=OFFLINE, location=5d4d7471f19e,37405,1733555609713; forceNewPlan=false, retain=false 2024-12-07T07:13:32,864 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:32,865 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN; state=OFFLINE, location=5d4d7471f19e,41097,1733555609760; forceNewPlan=false, retain=false 2024-12-07T07:13:32,867 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ffbaf4a5440811c93c49fb27af711a24, ASSIGN 2024-12-07T07:13:32,868 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ffbaf4a5440811c93c49fb27af711a24, ASSIGN; state=OFFLINE, location=5d4d7471f19e,34103,1733555609616; forceNewPlan=false, retain=false 2024-12-07T07:13:32,870 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=37e2dfda6af2ae02ec7c14d6781d4503, ASSIGN 2024-12-07T07:13:32,872 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=37e2dfda6af2ae02ec7c14d6781d4503, ASSIGN; state=OFFLINE, location=5d4d7471f19e,34103,1733555609616; forceNewPlan=false, retain=false 2024-12-07T07:13:32,873 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c709bd9f88fc6dff6480583f422247fa, ASSIGN 2024-12-07T07:13:32,875 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c709bd9f88fc6dff6480583f422247fa, ASSIGN; state=OFFLINE, location=5d4d7471f19e,37405,1733555609713; forceNewPlan=false, retain=false 2024-12-07T07:13:32,876 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:32,878 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN; state=OFFLINE, location=5d4d7471f19e,41097,1733555609760; forceNewPlan=false, retain=false 2024-12-07T07:13:32,879 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=45e7ad8954e0a4d51f932a71d965426b, ASSIGN 2024-12-07T07:13:32,880 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=45e7ad8954e0a4d51f932a71d965426b, ASSIGN; state=OFFLINE, location=5d4d7471f19e,34103,1733555609616; forceNewPlan=false, retain=false 2024-12-07T07:13:32,882 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c8344bda7e0c92708aba9a317b54627, ASSIGN 2024-12-07T07:13:32,883 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4c8344bda7e0c92708aba9a317b54627, ASSIGN; state=OFFLINE, location=5d4d7471f19e,37405,1733555609713; forceNewPlan=false, retain=false 2024-12-07T07:13:32,884 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:32,886 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN; state=OFFLINE, location=5d4d7471f19e,41097,1733555609760; forceNewPlan=false, retain=false 2024-12-07T07:13:32,887 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a24f56f7916756473277b23c11f5ce84, ASSIGN 2024-12-07T07:13:32,889 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a24f56f7916756473277b23c11f5ce84, ASSIGN; state=OFFLINE, location=5d4d7471f19e,37405,1733555609713; forceNewPlan=false, retain=false 2024-12-07T07:13:32,890 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51e3bd49877ea2b244de43a10f4eaa37, ASSIGN 2024-12-07T07:13:32,891 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=51e3bd49877ea2b244de43a10f4eaa37, ASSIGN; state=OFFLINE, location=5d4d7471f19e,34103,1733555609616; forceNewPlan=false, retain=false 2024-12-07T07:13:32,893 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:32,895 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN; state=OFFLINE, location=5d4d7471f19e,41097,1733555609760; forceNewPlan=false, retain=false 2024-12-07T07:13:32,896 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=caa1b434450bd250c5e2792eafa50df0, ASSIGN 2024-12-07T07:13:32,897 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=caa1b434450bd250c5e2792eafa50df0, ASSIGN; state=OFFLINE, location=5d4d7471f19e,37405,1733555609713; forceNewPlan=false, retain=false 2024-12-07T07:13:32,899 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5101f15d587bc41cd49e68708e4a5ed2, ASSIGN 2024-12-07T07:13:32,900 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5101f15d587bc41cd49e68708e4a5ed2, ASSIGN; state=OFFLINE, location=5d4d7471f19e,34103,1733555609616; forceNewPlan=false, retain=false 2024-12-07T07:13:32,901 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:32,903 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN; state=OFFLINE, location=5d4d7471f19e,41097,1733555609760; forceNewPlan=false, retain=false 2024-12-07T07:13:32,904 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6aadffb1e0d6160805fffee563de94ca, ASSIGN 2024-12-07T07:13:32,906 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6aadffb1e0d6160805fffee563de94ca, ASSIGN; state=OFFLINE, location=5d4d7471f19e,34103,1733555609616; forceNewPlan=false, retain=false 2024-12-07T07:13:32,907 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=764f719c1b7c5872a4cc2b9d21f2dc24, ASSIGN 2024-12-07T07:13:32,909 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=764f719c1b7c5872a4cc2b9d21f2dc24, ASSIGN; state=OFFLINE, location=5d4d7471f19e,37405,1733555609713; forceNewPlan=false, retain=false 2024-12-07T07:13:32,910 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cf320858c98aa00078ada211b555e73e, ASSIGN 2024-12-07T07:13:32,912 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cf320858c98aa00078ada211b555e73e, ASSIGN; state=OFFLINE, location=5d4d7471f19e,34103,1733555609616; forceNewPlan=false, retain=false 2024-12-07T07:13:32,913 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=58e2442015eb81a1979c8a3c79c8b4a4, ASSIGN 2024-12-07T07:13:32,915 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=58e2442015eb81a1979c8a3c79c8b4a4, ASSIGN; state=OFFLINE, location=5d4d7471f19e,37405,1733555609713; forceNewPlan=false, retain=false 2024-12-07T07:13:32,916 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:32,917 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN; state=OFFLINE, location=5d4d7471f19e,41097,1733555609760; forceNewPlan=false, retain=false 2024-12-07T07:13:32,999 INFO [5d4d7471f19e:46437 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-12-07T07:13:33,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,001 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,002 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,004 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN because future has completed 2024-12-07T07:13:33,005 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,41097,1733555609760}] 2024-12-07T07:13:33,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN because future has completed 2024-12-07T07:13:33,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN because future has completed 2024-12-07T07:13:33,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,41097,1733555609760}] 2024-12-07T07:13:33,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN because future has completed 2024-12-07T07:13:33,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN because future has completed 2024-12-07T07:13:33,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN because future has completed 2024-12-07T07:13:33,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN because future has completed 2024-12-07T07:13:33,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,41097,1733555609760}] 2024-12-07T07:13:33,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfa28a210d48360878dfe1333fd8e26, server=5d4d7471f19e,41097,1733555609760}] 2024-12-07T07:13:33,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,41097,1733555609760}] 2024-12-07T07:13:33,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,41097,1733555609760}] 2024-12-07T07:13:33,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,41097,1733555609760}] 2024-12-07T07:13:33,051 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,052 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,053 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=99de37e290b4841263ba62805b625d97, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,054 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,054 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c709bd9f88fc6dff6480583f422247fa, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=4c8344bda7e0c92708aba9a317b54627, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a24f56f7916756473277b23c11f5ce84, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN because future has completed 2024-12-07T07:13:33,056 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=caa1b434450bd250c5e2792eafa50df0, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,41097,1733555609760}] 2024-12-07T07:13:33,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=abe9b4a81ffb7c674120d136c6ab6231, ASSIGN because future has completed 2024-12-07T07:13:33,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=99de37e290b4841263ba62805b625d97, ASSIGN because future has completed 2024-12-07T07:13:33,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=da5554a80a6c43e6ba1b1014d9f4023a, ASSIGN because future has completed 2024-12-07T07:13:33,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure da5554a80a6c43e6ba1b1014d9f4023a, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:33,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c709bd9f88fc6dff6480583f422247fa, ASSIGN because future has completed 2024-12-07T07:13:33,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4c8344bda7e0c92708aba9a317b54627, ASSIGN because future has completed 2024-12-07T07:13:33,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a24f56f7916756473277b23c11f5ce84, ASSIGN because future has completed 2024-12-07T07:13:33,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=caa1b434450bd250c5e2792eafa50df0, ASSIGN because future has completed 2024-12-07T07:13:33,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure caa1b434450bd250c5e2792eafa50df0, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:33,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24f56f7916756473277b23c11f5ce84, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:33,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c8344bda7e0c92708aba9a317b54627, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:33,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c709bd9f88fc6dff6480583f422247fa, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:33,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99de37e290b4841263ba62805b625d97, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:33,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure abe9b4a81ffb7c674120d136c6ab6231, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:33,094 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,094 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,095 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=d60e3c4175109e97854d2b36c43a459f, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,096 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5369f3b9703dae7de70d6556989c74a8, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,096 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=ffbaf4a5440811c93c49fb27af711a24, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,097 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=764f719c1b7c5872a4cc2b9d21f2dc24, ASSIGN because future has completed 2024-12-07T07:13:33,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 764f719c1b7c5872a4cc2b9d21f2dc24, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:33,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=58e2442015eb81a1979c8a3c79c8b4a4, ASSIGN because future has completed 2024-12-07T07:13:33,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d60e3c4175109e97854d2b36c43a459f, ASSIGN because future has completed 2024-12-07T07:13:33,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5369f3b9703dae7de70d6556989c74a8, ASSIGN because future has completed 2024-12-07T07:13:33,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ffbaf4a5440811c93c49fb27af711a24, ASSIGN because future has completed 2024-12-07T07:13:33,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffbaf4a5440811c93c49fb27af711a24, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:33,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=37e2dfda6af2ae02ec7c14d6781d4503, ASSIGN because future has completed 2024-12-07T07:13:33,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37e2dfda6af2ae02ec7c14d6781d4503, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:33,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5369f3b9703dae7de70d6556989c74a8, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:33,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure d60e3c4175109e97854d2b36c43a459f, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:33,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58e2442015eb81a1979c8a3c79c8b4a4, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:33,125 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=45e7ad8954e0a4d51f932a71d965426b, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,127 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=6aadffb1e0d6160805fffee563de94ca, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,128 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cf320858c98aa00078ada211b555e73e, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=45e7ad8954e0a4d51f932a71d965426b, ASSIGN because future has completed 2024-12-07T07:13:33,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e7ad8954e0a4d51f932a71d965426b, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:33,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=51e3bd49877ea2b244de43a10f4eaa37, ASSIGN because future has completed 2024-12-07T07:13:33,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5101f15d587bc41cd49e68708e4a5ed2, ASSIGN because future has completed 2024-12-07T07:13:33,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6aadffb1e0d6160805fffee563de94ca, ASSIGN because future has completed 2024-12-07T07:13:33,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6aadffb1e0d6160805fffee563de94ca, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:33,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cf320858c98aa00078ada211b555e73e, ASSIGN because future has completed 2024-12-07T07:13:33,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf320858c98aa00078ada211b555e73e, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:33,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5101f15d587bc41cd49e68708e4a5ed2, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:33,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51e3bd49877ea2b244de43a10f4eaa37, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:33,167 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:33,167 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => acd8e0c860fdbce58ce803c15220afb0, NAME => 'test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-07T07:13:33,167 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,167 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,167 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,167 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,170 INFO [StoreOpener-acd8e0c860fdbce58ce803c15220afb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,172 INFO [StoreOpener-acd8e0c860fdbce58ce803c15220afb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acd8e0c860fdbce58ce803c15220afb0 columnFamilyName family 2024-12-07T07:13:33,172 DEBUG [StoreOpener-acd8e0c860fdbce58ce803c15220afb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,173 INFO [StoreOpener-acd8e0c860fdbce58ce803c15220afb0-1 {}] regionserver.HStore(327): Store=acd8e0c860fdbce58ce803c15220afb0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,173 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,175 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,175 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,176 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,176 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,178 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,182 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/acd8e0c860fdbce58ce803c15220afb0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,182 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened acd8e0c860fdbce58ce803c15220afb0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69564799, jitterRate=0.036596283316612244}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,183 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:33,184 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for acd8e0c860fdbce58ce803c15220afb0: Running coprocessor pre-open hook at 1733555613168Writing region info on filesystem at 1733555613168Initializing all the Stores at 1733555613169 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613169Cleaning up temporary data from old regions at 1733555613176 (+7 ms)Running coprocessor post-open hooks at 1733555613183 (+7 ms)Region opened successfully at 1733555613184 (+1 ms) 2024-12-07T07:13:33,186 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0., pid=33, masterSystemTime=1733555613161 2024-12-07T07:13:33,189 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:33,189 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:33,189 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:33,190 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => d07cc4fd0d69fc55b90b071c1b0400c7, NAME => 'test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-07T07:13:33,190 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=acd8e0c860fdbce58ce803c15220afb0, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,190 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,190 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,190 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,190 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,193 INFO [StoreOpener-d07cc4fd0d69fc55b90b071c1b0400c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,41097,1733555609760 because future has completed 2024-12-07T07:13:33,195 INFO [StoreOpener-d07cc4fd0d69fc55b90b071c1b0400c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d07cc4fd0d69fc55b90b071c1b0400c7 columnFamilyName family 2024-12-07T07:13:33,195 DEBUG [StoreOpener-d07cc4fd0d69fc55b90b071c1b0400c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,196 INFO [StoreOpener-d07cc4fd0d69fc55b90b071c1b0400c7-1 {}] regionserver.HStore(327): Store=d07cc4fd0d69fc55b90b071c1b0400c7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,196 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,198 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=10 2024-12-07T07:13:33,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,41097,1733555609760 in 173 msec 2024-12-07T07:13:33,199 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,199 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,199 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN in 362 msec 2024-12-07T07:13:33,202 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,205 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/d07cc4fd0d69fc55b90b071c1b0400c7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,206 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened d07cc4fd0d69fc55b90b071c1b0400c7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66626016, jitterRate=-0.007194995880126953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,206 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:33,206 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for d07cc4fd0d69fc55b90b071c1b0400c7: Running coprocessor pre-open hook at 1733555613190Writing region info on filesystem at 1733555613190Initializing all the Stores at 1733555613192 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613192Cleaning up temporary data from old regions at 1733555613200 (+8 ms)Running coprocessor post-open hooks at 1733555613206 (+6 ms)Region opened successfully at 1733555613206 2024-12-07T07:13:33,207 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7., pid=31, masterSystemTime=1733555613161 2024-12-07T07:13:33,210 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:33,210 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:33,210 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:33,211 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => c1e55fd1b7105eca2dcbd1d5897f7f5a, NAME => 'test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-07T07:13:33,211 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,211 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,211 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,211 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,211 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,214 INFO [StoreOpener-c1e55fd1b7105eca2dcbd1d5897f7f5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,41097,1733555609760 because future has completed 2024-12-07T07:13:33,216 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T07:13:33,216 INFO [StoreOpener-c1e55fd1b7105eca2dcbd1d5897f7f5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1e55fd1b7105eca2dcbd1d5897f7f5a columnFamilyName family 2024-12-07T07:13:33,216 DEBUG [StoreOpener-c1e55fd1b7105eca2dcbd1d5897f7f5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,217 INFO [StoreOpener-c1e55fd1b7105eca2dcbd1d5897f7f5a-1 {}] regionserver.HStore(327): Store=c1e55fd1b7105eca2dcbd1d5897f7f5a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,217 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42205, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T07:13:33,219 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,220 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,220 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,221 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-12-07T07:13:33,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,41097,1733555609760 in 210 msec 2024-12-07T07:13:33,224 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN in 386 msec 2024-12-07T07:13:33,224 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:33,225 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 99de37e290b4841263ba62805b625d97, NAME => 'test,www,1733555612106.99de37e290b4841263ba62805b625d97.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-07T07:13:33,225 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,225 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,www,1733555612106.99de37e290b4841263ba62805b625d97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,226 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,226 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,227 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/c1e55fd1b7105eca2dcbd1d5897f7f5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,227 INFO [StoreOpener-99de37e290b4841263ba62805b625d97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,228 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened c1e55fd1b7105eca2dcbd1d5897f7f5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63242645, jitterRate=-0.05761115252971649}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,228 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:33,228 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for c1e55fd1b7105eca2dcbd1d5897f7f5a: Running coprocessor pre-open hook at 1733555613211Writing region info on filesystem at 1733555613211Initializing all the Stores at 1733555613213 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613213Cleaning up temporary data from old regions at 1733555613221 (+8 ms)Running coprocessor post-open hooks at 1733555613228 (+7 ms)Region opened successfully at 1733555613228 2024-12-07T07:13:33,229 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a., pid=36, masterSystemTime=1733555613161 2024-12-07T07:13:33,231 INFO [StoreOpener-99de37e290b4841263ba62805b625d97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99de37e290b4841263ba62805b625d97 columnFamilyName family 2024-12-07T07:13:33,231 DEBUG [StoreOpener-99de37e290b4841263ba62805b625d97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,232 INFO [StoreOpener-99de37e290b4841263ba62805b625d97-1 {}] regionserver.HStore(327): Store=99de37e290b4841263ba62805b625d97/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,232 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:33,232 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:33,232 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,232 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:33,233 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => ff1807b6b64d6c4ca2c6c6d7a2cafbd3, NAME => 'test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-07T07:13:33,233 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,233 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,233 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,233 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,233 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,233 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,234 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,235 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,235 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,235 INFO [StoreOpener-ff1807b6b64d6c4ca2c6c6d7a2cafbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,41097,1733555609760 because future has completed 2024-12-07T07:13:33,237 INFO [StoreOpener-ff1807b6b64d6c4ca2c6c6d7a2cafbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff1807b6b64d6c4ca2c6c6d7a2cafbd3 columnFamilyName family 2024-12-07T07:13:33,237 DEBUG [StoreOpener-ff1807b6b64d6c4ca2c6c6d7a2cafbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,238 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,238 INFO [StoreOpener-ff1807b6b64d6c4ca2c6c6d7a2cafbd3-1 {}] regionserver.HStore(327): Store=ff1807b6b64d6c4ca2c6c6d7a2cafbd3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,238 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,240 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,240 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,241 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,241 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,241 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/99de37e290b4841263ba62805b625d97/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=19 2024-12-07T07:13:33,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,41097,1733555609760 in 198 msec 2024-12-07T07:13:33,242 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 99de37e290b4841263ba62805b625d97; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67348721, jitterRate=0.0035741478204727173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,242 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99de37e290b4841263ba62805b625d97 2024-12-07T07:13:33,242 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 99de37e290b4841263ba62805b625d97: Running coprocessor pre-open hook at 1733555613226Writing region info on filesystem at 1733555613226Initializing all the Stores at 1733555613227 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613227Cleaning up temporary data from old regions at 1733555613235 (+8 ms)Running coprocessor post-open hooks at 1733555613242 (+7 ms)Region opened successfully at 1733555613242 2024-12-07T07:13:33,244 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,244 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733555612106.99de37e290b4841263ba62805b625d97., pid=44, masterSystemTime=1733555613215 2024-12-07T07:13:33,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN in 406 msec 2024-12-07T07:13:33,248 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:33,248 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:33,248 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:33,249 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 58e2442015eb81a1979c8a3c79c8b4a4, NAME => 'test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-07T07:13:33,249 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,249 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,249 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,249 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ff1807b6b64d6c4ca2c6c6d7a2cafbd3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,249 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=99de37e290b4841263ba62805b625d97, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,250 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened ff1807b6b64d6c4ca2c6c6d7a2cafbd3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68497457, jitterRate=0.0206916481256485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,250 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:33,250 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for ff1807b6b64d6c4ca2c6c6d7a2cafbd3: Running coprocessor pre-open hook at 1733555613233Writing region info on filesystem at 1733555613233Initializing all the Stores at 1733555613235 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613235Cleaning up temporary data from old regions at 1733555613241 (+6 ms)Running coprocessor post-open hooks at 1733555613250 (+9 ms)Region opened successfully at 1733555613250 2024-12-07T07:13:33,252 INFO [StoreOpener-58e2442015eb81a1979c8a3c79c8b4a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99de37e290b4841263ba62805b625d97, server=5d4d7471f19e,37405,1733555609713 because future has completed 2024-12-07T07:13:33,252 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3., pid=38, masterSystemTime=1733555613161 2024-12-07T07:13:33,255 INFO [StoreOpener-58e2442015eb81a1979c8a3c79c8b4a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58e2442015eb81a1979c8a3c79c8b4a4 columnFamilyName family 2024-12-07T07:13:33,255 DEBUG [StoreOpener-58e2442015eb81a1979c8a3c79c8b4a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,256 INFO [StoreOpener-58e2442015eb81a1979c8a3c79c8b4a4-1 {}] regionserver.HStore(327): Store=58e2442015eb81a1979c8a3c79c8b4a4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,256 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,257 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T07:13:33,257 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:33,257 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:33,257 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:33,258 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => ab678da1dd813ac72686a84c4f22fdc0, NAME => 'test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-07T07:13:33,258 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,258 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=27 2024-12-07T07:13:33,258 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,258 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 99de37e290b4841263ba62805b625d97, server=5d4d7471f19e,37405,1733555609713 in 169 msec 2024-12-07T07:13:33,259 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38059, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T07:13:33,260 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,261 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,261 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,261 INFO [StoreOpener-ab678da1dd813ac72686a84c4f22fdc0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=99de37e290b4841263ba62805b625d97, ASSIGN in 423 msec 2024-12-07T07:13:33,264 INFO [StoreOpener-ab678da1dd813ac72686a84c4f22fdc0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab678da1dd813ac72686a84c4f22fdc0 columnFamilyName family 2024-12-07T07:13:33,264 DEBUG [StoreOpener-ab678da1dd813ac72686a84c4f22fdc0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,265 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,265 INFO [StoreOpener-ab678da1dd813ac72686a84c4f22fdc0-1 {}] regionserver.HStore(327): Store=ab678da1dd813ac72686a84c4f22fdc0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,265 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,265 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,266 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,267 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,268 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,268 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,268 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:33,268 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 5369f3b9703dae7de70d6556989c74a8, NAME => 'test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-07T07:13:33,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,41097,1733555609760 because future has completed 2024-12-07T07:13:33,269 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,269 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,269 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,269 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,271 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,272 INFO [StoreOpener-5369f3b9703dae7de70d6556989c74a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,274 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ab678da1dd813ac72686a84c4f22fdc0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,274 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/58e2442015eb81a1979c8a3c79c8b4a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,274 INFO [StoreOpener-5369f3b9703dae7de70d6556989c74a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5369f3b9703dae7de70d6556989c74a8 columnFamilyName family 2024-12-07T07:13:33,275 DEBUG [StoreOpener-5369f3b9703dae7de70d6556989c74a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,275 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened ab678da1dd813ac72686a84c4f22fdc0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65529746, jitterRate=-0.023530691862106323}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,275 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:33,275 INFO [StoreOpener-5369f3b9703dae7de70d6556989c74a8-1 {}] regionserver.HStore(327): Store=5369f3b9703dae7de70d6556989c74a8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,276 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for ab678da1dd813ac72686a84c4f22fdc0: Running coprocessor pre-open hook at 1733555613258Writing region info on filesystem at 1733555613258Initializing all the Stores at 1733555613260 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613260Cleaning up temporary data from old regions at 1733555613268 (+8 ms)Running coprocessor post-open hooks at 1733555613276 (+8 ms)Region opened successfully at 1733555613276 2024-12-07T07:13:33,276 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=5 2024-12-07T07:13:33,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,41097,1733555609760 in 214 msec 2024-12-07T07:13:33,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T07:13:33,277 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0., pid=32, masterSystemTime=1733555613161 2024-12-07T07:13:33,277 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 58e2442015eb81a1979c8a3c79c8b4a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68916376, jitterRate=0.026934027671813965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,277 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:33,278 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,278 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,279 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 58e2442015eb81a1979c8a3c79c8b4a4: Running coprocessor pre-open hook at 1733555613250Writing region info on filesystem at 1733555613250Initializing all the Stores at 1733555613251 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613251Cleaning up temporary data from old regions at 1733555613261 (+10 ms)Running coprocessor post-open hooks at 1733555613278 (+17 ms)Region opened successfully at 1733555613278 2024-12-07T07:13:33,279 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,279 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,280 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:33,280 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4., pid=51, masterSystemTime=1733555613215 2024-12-07T07:13:33,280 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:33,280 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:33,280 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 2cfa28a210d48360878dfe1333fd8e26, NAME => 'test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-07T07:13:33,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN in 440 msec 2024-12-07T07:13:33,281 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,281 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,281 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,281 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,282 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,283 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:33,283 INFO [StoreOpener-2cfa28a210d48360878dfe1333fd8e26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,283 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:33,283 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:33,283 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => caa1b434450bd250c5e2792eafa50df0, NAME => 'test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-07T07:13:33,284 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,284 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,284 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,284 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,285 INFO [StoreOpener-2cfa28a210d48360878dfe1333fd8e26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cfa28a210d48360878dfe1333fd8e26 columnFamilyName family 2024-12-07T07:13:33,285 DEBUG [StoreOpener-2cfa28a210d48360878dfe1333fd8e26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,286 INFO [StoreOpener-2cfa28a210d48360878dfe1333fd8e26-1 {}] regionserver.HStore(327): Store=2cfa28a210d48360878dfe1333fd8e26/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,286 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,41097,1733555609760 because future has completed 2024-12-07T07:13:33,287 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,287 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,288 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,289 INFO [StoreOpener-caa1b434450bd250c5e2792eafa50df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,289 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,289 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,289 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/5369f3b9703dae7de70d6556989c74a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,290 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 5369f3b9703dae7de70d6556989c74a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65569579, jitterRate=-0.02293713390827179}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,290 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:33,290 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 5369f3b9703dae7de70d6556989c74a8: Running coprocessor pre-open hook at 1733555613269Writing region info on filesystem at 1733555613269Initializing all the Stores at 1733555613271 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613271Cleaning up temporary data from old regions at 1733555613279 (+8 ms)Running coprocessor post-open hooks at 1733555613290 (+11 ms)Region opened successfully at 1733555613290 2024-12-07T07:13:33,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58e2442015eb81a1979c8a3c79c8b4a4, server=5d4d7471f19e,37405,1733555609713 because future has completed 2024-12-07T07:13:33,292 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8., pid=49, masterSystemTime=1733555613256 2024-12-07T07:13:33,292 INFO [StoreOpener-caa1b434450bd250c5e2792eafa50df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caa1b434450bd250c5e2792eafa50df0 columnFamilyName family 2024-12-07T07:13:33,292 DEBUG [StoreOpener-caa1b434450bd250c5e2792eafa50df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,293 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,293 INFO [StoreOpener-caa1b434450bd250c5e2792eafa50df0-1 {}] regionserver.HStore(327): Store=caa1b434450bd250c5e2792eafa50df0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,294 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,295 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,296 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,297 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:33,297 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:33,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=23 2024-12-07T07:13:33,297 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:33,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,41097,1733555609760 in 278 msec 2024-12-07T07:13:33,297 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 6aadffb1e0d6160805fffee563de94ca, NAME => 'test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-07T07:13:33,298 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,298 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,298 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/2cfa28a210d48360878dfe1333fd8e26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,298 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,298 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,298 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,298 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,299 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 2cfa28a210d48360878dfe1333fd8e26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72228054, jitterRate=0.07628187537193298}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,299 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:33,300 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 2cfa28a210d48360878dfe1333fd8e26: Running coprocessor pre-open hook at 1733555613281Writing region info on filesystem at 1733555613281Initializing all the Stores at 1733555613282 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613282Cleaning up temporary data from old regions at 1733555613289 (+7 ms)Running coprocessor post-open hooks at 1733555613300 (+11 ms)Region opened successfully at 1733555613300 2024-12-07T07:13:33,301 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26., pid=34, masterSystemTime=1733555613161 2024-12-07T07:13:33,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN in 461 msec 2024-12-07T07:13:33,302 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,305 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:33,305 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:33,305 INFO [StoreOpener-6aadffb1e0d6160805fffee563de94ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,305 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:33,305 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d2616a94cd0a6eca969a2466add4db9, NAME => 'test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-07T07:13:33,306 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,306 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,306 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,306 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,308 INFO [StoreOpener-6aadffb1e0d6160805fffee563de94ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aadffb1e0d6160805fffee563de94ca columnFamilyName family 2024-12-07T07:13:33,309 DEBUG [StoreOpener-6aadffb1e0d6160805fffee563de94ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5369f3b9703dae7de70d6556989c74a8, server=5d4d7471f19e,34103,1733555609616 because future has completed 2024-12-07T07:13:33,309 INFO [StoreOpener-6aadffb1e0d6160805fffee563de94ca-1 {}] regionserver.HStore(327): Store=6aadffb1e0d6160805fffee563de94ca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,309 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,310 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=2cfa28a210d48360878dfe1333fd8e26, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,310 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,311 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,312 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,312 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,313 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/caa1b434450bd250c5e2792eafa50df0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfa28a210d48360878dfe1333fd8e26, server=5d4d7471f19e,41097,1733555609760 because future has completed 2024-12-07T07:13:33,314 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened caa1b434450bd250c5e2792eafa50df0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67870033, jitterRate=0.011342301964759827}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,314 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:33,314 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for caa1b434450bd250c5e2792eafa50df0: Running coprocessor pre-open hook at 1733555613284Writing region info on filesystem at 1733555613284Initializing all the Stores at 1733555613286 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613286Cleaning up temporary data from old regions at 1733555613298 (+12 ms)Running coprocessor post-open hooks at 1733555613314 (+16 ms)Region opened successfully at 1733555613314 2024-12-07T07:13:33,316 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0., pid=40, masterSystemTime=1733555613215 2024-12-07T07:13:33,317 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,317 INFO [StoreOpener-9d2616a94cd0a6eca969a2466add4db9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,320 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:33,320 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:33,320 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:33,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=25 2024-12-07T07:13:33,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 5369f3b9703dae7de70d6556989c74a8, server=5d4d7471f19e,34103,1733555609616 in 201 msec 2024-12-07T07:13:33,320 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => abe9b4a81ffb7c674120d136c6ab6231, NAME => 'test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-07T07:13:33,320 INFO [StoreOpener-9d2616a94cd0a6eca969a2466add4db9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d2616a94cd0a6eca969a2466add4db9 columnFamilyName family 2024-12-07T07:13:33,321 DEBUG [StoreOpener-9d2616a94cd0a6eca969a2466add4db9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,321 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,321 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,321 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,321 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,322 INFO [StoreOpener-9d2616a94cd0a6eca969a2466add4db9-1 {}] regionserver.HStore(327): Store=9d2616a94cd0a6eca969a2466add4db9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,322 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,322 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/6aadffb1e0d6160805fffee563de94ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,326 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,326 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5d4d7471f19e,37405,1733555609713, table=test, region=58e2442015eb81a1979c8a3c79c8b4a4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T07:13:33,327 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,327 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 6aadffb1e0d6160805fffee563de94ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74527073, jitterRate=0.11053992807865143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,328 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:33,328 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 6aadffb1e0d6160805fffee563de94ca: Running coprocessor pre-open hook at 1733555613299Writing region info on filesystem at 1733555613299Initializing all the Stores at 1733555613301 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613301Cleaning up temporary data from old regions at 1733555613312 (+11 ms)Running coprocessor post-open hooks at 1733555613328 (+16 ms)Region opened successfully at 1733555613328 2024-12-07T07:13:33,328 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,328 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5369f3b9703dae7de70d6556989c74a8, ASSIGN in 485 msec 2024-12-07T07:13:33,329 INFO [StoreOpener-abe9b4a81ffb7c674120d136c6ab6231-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,329 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca., pid=53, masterSystemTime=1733555613256 2024-12-07T07:13:33,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,333 INFO [StoreOpener-abe9b4a81ffb7c674120d136c6ab6231-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abe9b4a81ffb7c674120d136c6ab6231 columnFamilyName family 2024-12-07T07:13:33,333 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:33,333 DEBUG [StoreOpener-abe9b4a81ffb7c674120d136c6ab6231-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,334 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:33,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure caa1b434450bd250c5e2792eafa50df0, server=5d4d7471f19e,37405,1733555609713 because future has completed 2024-12-07T07:13:33,334 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:33,334 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,334 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 51e3bd49877ea2b244de43a10f4eaa37, NAME => 'test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-07T07:13:33,334 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,334 INFO [StoreOpener-abe9b4a81ffb7c674120d136c6ab6231-1 {}] regionserver.HStore(327): Store=abe9b4a81ffb7c674120d136c6ab6231/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,334 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,335 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,335 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,335 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,336 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,336 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,337 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,337 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,341 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,341 INFO [StoreOpener-51e3bd49877ea2b244de43a10f4eaa37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,341 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/9d2616a94cd0a6eca969a2466add4db9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,342 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 9d2616a94cd0a6eca969a2466add4db9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73259226, jitterRate=0.09164753556251526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,342 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:33,342 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 9d2616a94cd0a6eca969a2466add4db9: Running coprocessor pre-open hook at 1733555613306Writing region info on filesystem at 1733555613306Initializing all the Stores at 1733555613308 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613308Cleaning up temporary data from old regions at 1733555613328 (+20 ms)Running coprocessor post-open hooks at 1733555613342 (+14 ms)Region opened successfully at 1733555613342 2024-12-07T07:13:33,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=12 2024-12-07T07:13:33,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure caa1b434450bd250c5e2792eafa50df0, server=5d4d7471f19e,37405,1733555609713 in 270 msec 2024-12-07T07:13:33,344 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9., pid=37, masterSystemTime=1733555613161 2024-12-07T07:13:33,345 INFO [StoreOpener-51e3bd49877ea2b244de43a10f4eaa37-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51e3bd49877ea2b244de43a10f4eaa37 columnFamilyName family 2024-12-07T07:13:33,345 DEBUG [StoreOpener-51e3bd49877ea2b244de43a10f4eaa37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,346 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:33,347 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:33,347 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:33,347 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => fe956aa50065b9ae6300ba6efb3d715d, NAME => 'test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-07T07:13:33,348 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,348 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,348 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=caa1b434450bd250c5e2792eafa50df0, ASSIGN in 508 msec 2024-12-07T07:13:33,348 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,349 INFO [StoreOpener-51e3bd49877ea2b244de43a10f4eaa37-1 {}] regionserver.HStore(327): Store=51e3bd49877ea2b244de43a10f4eaa37/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,349 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,350 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,351 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,351 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=9d2616a94cd0a6eca969a2466add4db9, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,352 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,352 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,352 INFO [StoreOpener-fe956aa50065b9ae6300ba6efb3d715d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,357 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5d4d7471f19e,41097,1733555609760, table=test, region=2cfa28a210d48360878dfe1333fd8e26. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T07:13:33,357 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5d4d7471f19e,41097,1733555609760, table=test, region=9d2616a94cd0a6eca969a2466add4db9. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T07:13:33,358 INFO [StoreOpener-fe956aa50065b9ae6300ba6efb3d715d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe956aa50065b9ae6300ba6efb3d715d columnFamilyName family 2024-12-07T07:13:33,358 DEBUG [StoreOpener-fe956aa50065b9ae6300ba6efb3d715d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,359 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/abe9b4a81ffb7c674120d136c6ab6231/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,41097,1733555609760 because future has completed 2024-12-07T07:13:33,360 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened abe9b4a81ffb7c674120d136c6ab6231; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69149453, jitterRate=0.030407145619392395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,360 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:33,360 INFO [StoreOpener-fe956aa50065b9ae6300ba6efb3d715d-1 {}] regionserver.HStore(327): Store=fe956aa50065b9ae6300ba6efb3d715d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,360 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,361 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,361 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for abe9b4a81ffb7c674120d136c6ab6231: Running coprocessor pre-open hook at 1733555613321Writing region info on filesystem at 1733555613321Initializing all the Stores at 1733555613327 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613327Cleaning up temporary data from old regions at 1733555613337 (+10 ms)Running coprocessor post-open hooks at 1733555613360 (+23 ms)Region opened successfully at 1733555613361 (+1 ms) 2024-12-07T07:13:33,361 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,361 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,362 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,363 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,363 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,364 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231., pid=45, masterSystemTime=1733555613215 2024-12-07T07:13:33,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6aadffb1e0d6160805fffee563de94ca, server=5d4d7471f19e,34103,1733555609616 because future has completed 2024-12-07T07:13:33,366 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,370 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:33,370 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:33,370 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:33,370 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c8344bda7e0c92708aba9a317b54627, NAME => 'test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-07T07:13:33,371 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,371 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,371 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,371 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,372 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/51e3bd49877ea2b244de43a10f4eaa37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=9 2024-12-07T07:13:33,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 6aadffb1e0d6160805fffee563de94ca, server=5d4d7471f19e,34103,1733555609616 in 234 msec 2024-12-07T07:13:33,373 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 51e3bd49877ea2b244de43a10f4eaa37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70385637, jitterRate=0.048827722668647766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,373 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:33,374 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 51e3bd49877ea2b244de43a10f4eaa37: Running coprocessor pre-open hook at 1733555613335Writing region info on filesystem at 1733555613335Initializing all the Stores at 1733555613337 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613337Cleaning up temporary data from old regions at 1733555613352 (+15 ms)Running coprocessor post-open hooks at 1733555613373 (+21 ms)Region opened successfully at 1733555613373 2024-12-07T07:13:33,375 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37., pid=56, masterSystemTime=1733555613256 2024-12-07T07:13:33,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6aadffb1e0d6160805fffee563de94ca, ASSIGN in 537 msec 2024-12-07T07:13:33,377 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/fe956aa50065b9ae6300ba6efb3d715d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,378 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:33,378 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:33,378 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:33,378 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened fe956aa50065b9ae6300ba6efb3d715d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68053579, jitterRate=0.014077350497245789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,378 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:33,378 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 5101f15d587bc41cd49e68708e4a5ed2, NAME => 'test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-07T07:13:33,378 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for fe956aa50065b9ae6300ba6efb3d715d: Running coprocessor pre-open hook at 1733555613348Writing region info on filesystem at 1733555613348Initializing all the Stores at 1733555613351 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613351Cleaning up temporary data from old regions at 1733555613363 (+12 ms)Running coprocessor post-open hooks at 1733555613378 (+15 ms)Region opened successfully at 1733555613378 2024-12-07T07:13:33,379 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,379 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,379 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,379 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,380 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d., pid=35, masterSystemTime=1733555613161 2024-12-07T07:13:33,380 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure abe9b4a81ffb7c674120d136c6ab6231, server=5d4d7471f19e,37405,1733555609713 because future has completed 2024-12-07T07:13:33,384 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:33,384 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:33,385 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51e3bd49877ea2b244de43a10f4eaa37, server=5d4d7471f19e,34103,1733555609616 because future has completed 2024-12-07T07:13:33,392 INFO [StoreOpener-4c8344bda7e0c92708aba9a317b54627-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=30 2024-12-07T07:13:33,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure abe9b4a81ffb7c674120d136c6ab6231, server=5d4d7471f19e,37405,1733555609713 in 300 msec 2024-12-07T07:13:33,395 INFO [StoreOpener-4c8344bda7e0c92708aba9a317b54627-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c8344bda7e0c92708aba9a317b54627 columnFamilyName family 2024-12-07T07:13:33,395 DEBUG [StoreOpener-4c8344bda7e0c92708aba9a317b54627-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,395 INFO [StoreOpener-4c8344bda7e0c92708aba9a317b54627-1 {}] regionserver.HStore(327): Store=4c8344bda7e0c92708aba9a317b54627/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,396 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=abe9b4a81ffb7c674120d136c6ab6231, ASSIGN in 557 msec 2024-12-07T07:13:33,397 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,398 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,399 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,399 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=14 2024-12-07T07:13:33,402 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 51e3bd49877ea2b244de43a10f4eaa37, server=5d4d7471f19e,34103,1733555609616 in 253 msec 2024-12-07T07:13:33,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=51e3bd49877ea2b244de43a10f4eaa37, ASSIGN in 567 msec 2024-12-07T07:13:33,407 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:33,408 INFO [StoreOpener-5101f15d587bc41cd49e68708e4a5ed2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,410 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/4c8344bda7e0c92708aba9a317b54627/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,411 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 4c8344bda7e0c92708aba9a317b54627; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73539682, jitterRate=0.095826655626297}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,412 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:33,412 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 4c8344bda7e0c92708aba9a317b54627: Running coprocessor pre-open hook at 1733555613372Writing region info on filesystem at 1733555613372Initializing all the Stores at 1733555613373 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613374 (+1 ms)Cleaning up temporary data from old regions at 1733555613399 (+25 ms)Running coprocessor post-open hooks at 1733555613412 (+13 ms)Region opened successfully at 1733555613412 2024-12-07T07:13:33,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,41097,1733555609760 because future has completed 2024-12-07T07:13:33,413 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627., pid=42, masterSystemTime=1733555613215 2024-12-07T07:13:33,413 INFO [StoreOpener-5101f15d587bc41cd49e68708e4a5ed2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5101f15d587bc41cd49e68708e4a5ed2 columnFamilyName family 2024-12-07T07:13:33,414 DEBUG [StoreOpener-5101f15d587bc41cd49e68708e4a5ed2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,415 INFO [StoreOpener-5101f15d587bc41cd49e68708e4a5ed2-1 {}] regionserver.HStore(327): Store=5101f15d587bc41cd49e68708e4a5ed2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,415 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,416 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,417 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,417 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:33,417 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:33,417 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:33,418 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 764f719c1b7c5872a4cc2b9d21f2dc24, NAME => 'test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-07T07:13:33,418 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,418 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,419 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=26 2024-12-07T07:13:33,419 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,41097,1733555609760 in 367 msec 2024-12-07T07:13:33,420 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,420 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN in 583 msec 2024-12-07T07:13:33,423 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,424 INFO [StoreOpener-764f719c1b7c5872a4cc2b9d21f2dc24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,425 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,432 INFO [StoreOpener-764f719c1b7c5872a4cc2b9d21f2dc24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 764f719c1b7c5872a4cc2b9d21f2dc24 columnFamilyName family 2024-12-07T07:13:33,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c8344bda7e0c92708aba9a317b54627, server=5d4d7471f19e,37405,1733555609713 because future has completed 2024-12-07T07:13:33,432 DEBUG [StoreOpener-764f719c1b7c5872a4cc2b9d21f2dc24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,433 INFO [StoreOpener-764f719c1b7c5872a4cc2b9d21f2dc24-1 {}] regionserver.HStore(327): Store=764f719c1b7c5872a4cc2b9d21f2dc24/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,433 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,433 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5d4d7471f19e,37405,1733555609713, table=test, region=58e2442015eb81a1979c8a3c79c8b4a4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T07:13:33,434 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5d4d7471f19e,37405,1733555609713, table=test, region=4c8344bda7e0c92708aba9a317b54627. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T07:13:33,434 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/5101f15d587bc41cd49e68708e4a5ed2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,435 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,435 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 5101f15d587bc41cd49e68708e4a5ed2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62615638, jitterRate=-0.06695428490638733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,435 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:33,435 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 5101f15d587bc41cd49e68708e4a5ed2: Running coprocessor pre-open hook at 1733555613379Writing region info on filesystem at 1733555613379Initializing all the Stores at 1733555613381 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613381Cleaning up temporary data from old regions at 1733555613420 (+39 ms)Running coprocessor post-open hooks at 1733555613435 (+15 ms)Region opened successfully at 1733555613435 2024-12-07T07:13:33,436 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,436 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,437 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,437 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2., pid=55, masterSystemTime=1733555613256 2024-12-07T07:13:33,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=16 2024-12-07T07:13:33,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,41097,1733555609760 in 399 msec 2024-12-07T07:13:33,440 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,440 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:33,441 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:33,441 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:33,441 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => ffbaf4a5440811c93c49fb27af711a24, NAME => 'test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-07T07:13:33,441 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,442 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,442 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,442 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN in 602 msec 2024-12-07T07:13:33,444 INFO [StoreOpener-ffbaf4a5440811c93c49fb27af711a24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,444 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/764f719c1b7c5872a4cc2b9d21f2dc24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,446 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,446 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 764f719c1b7c5872a4cc2b9d21f2dc24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58739228, jitterRate=-0.12471729516983032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,446 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:33,447 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 764f719c1b7c5872a4cc2b9d21f2dc24: Running coprocessor pre-open hook at 1733555613419Writing region info on filesystem at 1733555613419Initializing all the Stores at 1733555613421 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613421Cleaning up temporary data from old regions at 1733555613437 (+16 ms)Running coprocessor post-open hooks at 1733555613446 (+9 ms)Region opened successfully at 1733555613447 (+1 ms) 2024-12-07T07:13:33,448 INFO [StoreOpener-ffbaf4a5440811c93c49fb27af711a24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffbaf4a5440811c93c49fb27af711a24 columnFamilyName family 2024-12-07T07:13:33,448 DEBUG [StoreOpener-ffbaf4a5440811c93c49fb27af711a24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,449 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24., pid=46, masterSystemTime=1733555613215 2024-12-07T07:13:33,452 INFO [StoreOpener-ffbaf4a5440811c93c49fb27af711a24-1 {}] regionserver.HStore(327): Store=ffbaf4a5440811c93c49fb27af711a24/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,453 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,453 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:33,454 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:33,454 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:33,454 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => a24f56f7916756473277b23c11f5ce84, NAME => 'test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-07T07:13:33,454 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,454 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,455 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,455 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,455 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,455 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,456 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,456 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,458 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,460 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5101f15d587bc41cd49e68708e4a5ed2, server=5d4d7471f19e,34103,1733555609616 because future has completed 2024-12-07T07:13:33,462 INFO [StoreOpener-a24f56f7916756473277b23c11f5ce84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 764f719c1b7c5872a4cc2b9d21f2dc24, server=5d4d7471f19e,37405,1733555609713 because future has completed 2024-12-07T07:13:33,465 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5d4d7471f19e,41097,1733555609760, table=test, region=2cfa28a210d48360878dfe1333fd8e26. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T07:13:33,465 INFO [StoreOpener-a24f56f7916756473277b23c11f5ce84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a24f56f7916756473277b23c11f5ce84 columnFamilyName family 2024-12-07T07:13:33,465 DEBUG [StoreOpener-a24f56f7916756473277b23c11f5ce84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,466 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ffbaf4a5440811c93c49fb27af711a24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,466 INFO [StoreOpener-a24f56f7916756473277b23c11f5ce84-1 {}] regionserver.HStore(327): Store=a24f56f7916756473277b23c11f5ce84/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,466 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,467 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,467 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened ffbaf4a5440811c93c49fb27af711a24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72819253, jitterRate=0.08509142696857452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,468 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:33,469 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,469 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,470 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,468 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for ffbaf4a5440811c93c49fb27af711a24: Running coprocessor pre-open hook at 1733555613442Writing region info on filesystem at 1733555613442Initializing all the Stores at 1733555613444 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613444Cleaning up temporary data from old regions at 1733555613457 (+13 ms)Running coprocessor post-open hooks at 1733555613468 (+11 ms)Region opened successfully at 1733555613468 2024-12-07T07:13:33,472 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,473 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24., pid=47, masterSystemTime=1733555613256 2024-12-07T07:13:33,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=11 2024-12-07T07:13:33,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 5101f15d587bc41cd49e68708e4a5ed2, server=5d4d7471f19e,34103,1733555609616 in 325 msec 2024-12-07T07:13:33,476 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/a24f56f7916756473277b23c11f5ce84/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,477 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:33,477 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:33,477 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:33,477 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened a24f56f7916756473277b23c11f5ce84; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65877183, jitterRate=-0.018353477120399475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,478 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:33,478 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 37e2dfda6af2ae02ec7c14d6781d4503, NAME => 'test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-07T07:13:33,478 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for a24f56f7916756473277b23c11f5ce84: Running coprocessor pre-open hook at 1733555613455Writing region info on filesystem at 1733555613455Initializing all the Stores at 1733555613457 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613457Cleaning up temporary data from old regions at 1733555613470 (+13 ms)Running coprocessor post-open hooks at 1733555613478 (+8 ms)Region opened successfully at 1733555613478 2024-12-07T07:13:33,478 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,478 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,478 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,478 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,479 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84., pid=41, masterSystemTime=1733555613215 2024-12-07T07:13:33,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5101f15d587bc41cd49e68708e4a5ed2, ASSIGN in 638 msec 2024-12-07T07:13:33,480 INFO [StoreOpener-37e2dfda6af2ae02ec7c14d6781d4503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,482 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,483 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:33,483 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:33,483 INFO [StoreOpener-37e2dfda6af2ae02ec7c14d6781d4503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37e2dfda6af2ae02ec7c14d6781d4503 columnFamilyName family 2024-12-07T07:13:33,483 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:33,483 DEBUG [StoreOpener-37e2dfda6af2ae02ec7c14d6781d4503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,484 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => c709bd9f88fc6dff6480583f422247fa, NAME => 'test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-07T07:13:33,485 INFO [StoreOpener-37e2dfda6af2ae02ec7c14d6781d4503-1 {}] regionserver.HStore(327): Store=37e2dfda6af2ae02ec7c14d6781d4503/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,485 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,485 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,485 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,485 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,485 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffbaf4a5440811c93c49fb27af711a24, server=5d4d7471f19e,34103,1733555609616 because future has completed 2024-12-07T07:13:33,486 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,486 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,487 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,487 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,488 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,490 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,491 INFO [StoreOpener-c709bd9f88fc6dff6480583f422247fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a24f56f7916756473277b23c11f5ce84, server=5d4d7471f19e,37405,1733555609713 because future has completed 2024-12-07T07:13:33,494 INFO [StoreOpener-c709bd9f88fc6dff6480583f422247fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c709bd9f88fc6dff6480583f422247fa columnFamilyName family 2024-12-07T07:13:33,494 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/37e2dfda6af2ae02ec7c14d6781d4503/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,494 DEBUG [StoreOpener-c709bd9f88fc6dff6480583f422247fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,495 INFO [StoreOpener-c709bd9f88fc6dff6480583f422247fa-1 {}] regionserver.HStore(327): Store=c709bd9f88fc6dff6480583f422247fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,495 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 37e2dfda6af2ae02ec7c14d6781d4503; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66972995, jitterRate=-0.002024605870246887}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,495 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:33,495 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 37e2dfda6af2ae02ec7c14d6781d4503: Running coprocessor pre-open hook at 1733555613479Writing region info on filesystem at 1733555613479Initializing all the Stores at 1733555613480 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613480Cleaning up temporary data from old regions at 1733555613488 (+8 ms)Running coprocessor post-open hooks at 1733555613495 (+7 ms)Region opened successfully at 1733555613495 2024-12-07T07:13:33,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=22 2024-12-07T07:13:33,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure ffbaf4a5440811c93c49fb27af711a24, server=5d4d7471f19e,34103,1733555609616 in 387 msec 2024-12-07T07:13:33,496 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,496 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503., pid=48, masterSystemTime=1733555613256 2024-12-07T07:13:33,497 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,498 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,499 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,499 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ffbaf4a5440811c93c49fb27af711a24, ASSIGN in 660 msec 2024-12-07T07:13:33,500 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:33,500 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:33,500 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:33,500 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => cf320858c98aa00078ada211b555e73e, NAME => 'test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-07T07:13:33,501 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,501 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,501 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,501 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,503 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,505 INFO [StoreOpener-cf320858c98aa00078ada211b555e73e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37e2dfda6af2ae02ec7c14d6781d4503, server=5d4d7471f19e,34103,1733555609616 because future has completed 2024-12-07T07:13:33,508 INFO [StoreOpener-cf320858c98aa00078ada211b555e73e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf320858c98aa00078ada211b555e73e columnFamilyName family 2024-12-07T07:13:33,508 DEBUG [StoreOpener-cf320858c98aa00078ada211b555e73e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,508 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/c709bd9f88fc6dff6480583f422247fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,509 INFO [StoreOpener-cf320858c98aa00078ada211b555e73e-1 {}] regionserver.HStore(327): Store=cf320858c98aa00078ada211b555e73e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,509 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,509 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened c709bd9f88fc6dff6480583f422247fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62111220, jitterRate=-0.07447069883346558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,509 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:33,510 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for c709bd9f88fc6dff6480583f422247fa: Running coprocessor pre-open hook at 1733555613485Writing region info on filesystem at 1733555613485Initializing all the Stores at 1733555613487 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613487Cleaning up temporary data from old regions at 1733555613499 (+12 ms)Running coprocessor post-open hooks at 1733555613509 (+10 ms)Region opened successfully at 1733555613510 (+1 ms) 2024-12-07T07:13:33,512 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa., pid=43, masterSystemTime=1733555613215 2024-12-07T07:13:33,512 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=15 2024-12-07T07:13:33,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure a24f56f7916756473277b23c11f5ce84, server=5d4d7471f19e,37405,1733555609713 in 435 msec 2024-12-07T07:13:33,512 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,513 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,513 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,515 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,518 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:33,518 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:33,518 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:33,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a24f56f7916756473277b23c11f5ce84, ASSIGN in 676 msec 2024-12-07T07:13:33,518 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => da5554a80a6c43e6ba1b1014d9f4023a, NAME => 'test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-07T07:13:33,518 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/cf320858c98aa00078ada211b555e73e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,519 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,519 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,519 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,519 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened cf320858c98aa00078ada211b555e73e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67435098, jitterRate=0.004861265420913696}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,519 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:33,519 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,519 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for cf320858c98aa00078ada211b555e73e: Running coprocessor pre-open hook at 1733555613502Writing region info on filesystem at 1733555613502Initializing all the Stores at 1733555613503 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613504 (+1 ms)Cleaning up temporary data from old regions at 1733555613513 (+9 ms)Running coprocessor post-open hooks at 1733555613519 (+6 ms)Region opened successfully at 1733555613519 2024-12-07T07:13:33,520 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,520 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e., pid=54, masterSystemTime=1733555613256 2024-12-07T07:13:33,521 INFO [StoreOpener-da5554a80a6c43e6ba1b1014d9f4023a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,524 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:33,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c709bd9f88fc6dff6480583f422247fa, server=5d4d7471f19e,37405,1733555609713 because future has completed 2024-12-07T07:13:33,524 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:33,524 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:33,525 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => d60e3c4175109e97854d2b36c43a459f, NAME => 'test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-07T07:13:33,525 INFO [StoreOpener-da5554a80a6c43e6ba1b1014d9f4023a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da5554a80a6c43e6ba1b1014d9f4023a columnFamilyName family 2024-12-07T07:13:33,525 DEBUG [StoreOpener-da5554a80a6c43e6ba1b1014d9f4023a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,525 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,525 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,525 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,525 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,526 INFO [StoreOpener-da5554a80a6c43e6ba1b1014d9f4023a-1 {}] regionserver.HStore(327): Store=da5554a80a6c43e6ba1b1014d9f4023a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,526 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,527 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,528 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,528 INFO [StoreOpener-d60e3c4175109e97854d2b36c43a459f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,528 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,528 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=21 2024-12-07T07:13:33,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 37e2dfda6af2ae02ec7c14d6781d4503, server=5d4d7471f19e,34103,1733555609616 in 415 msec 2024-12-07T07:13:33,530 INFO [StoreOpener-d60e3c4175109e97854d2b36c43a459f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d60e3c4175109e97854d2b36c43a459f columnFamilyName family 2024-12-07T07:13:33,531 DEBUG [StoreOpener-d60e3c4175109e97854d2b36c43a459f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,531 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,531 INFO [StoreOpener-d60e3c4175109e97854d2b36c43a459f-1 {}] regionserver.HStore(327): Store=d60e3c4175109e97854d2b36c43a459f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,532 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=37e2dfda6af2ae02ec7c14d6781d4503, ASSIGN in 693 msec 2024-12-07T07:13:33,533 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,533 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,534 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,534 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,536 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/da5554a80a6c43e6ba1b1014d9f4023a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,537 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,538 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5d4d7471f19e,37405,1733555609713, table=test, region=58e2442015eb81a1979c8a3c79c8b4a4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T07:13:33,538 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5d4d7471f19e,37405,1733555609713, table=test, region=764f719c1b7c5872a4cc2b9d21f2dc24. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T07:13:33,538 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5d4d7471f19e,37405,1733555609713, table=test, region=4c8344bda7e0c92708aba9a317b54627. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T07:13:33,538 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened da5554a80a6c43e6ba1b1014d9f4023a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64907587, jitterRate=-0.03280158340930939}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,539 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:33,539 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for da5554a80a6c43e6ba1b1014d9f4023a: Running coprocessor pre-open hook at 1733555613519Writing region info on filesystem at 1733555613519Initializing all the Stores at 1733555613521 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613521Cleaning up temporary data from old regions at 1733555613528 (+7 ms)Running coprocessor post-open hooks at 1733555613539 (+11 ms)Region opened successfully at 1733555613539 2024-12-07T07:13:33,540 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a., pid=39, masterSystemTime=1733555613215 2024-12-07T07:13:33,541 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/d60e3c4175109e97854d2b36c43a459f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=20 2024-12-07T07:13:33,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure c709bd9f88fc6dff6480583f422247fa, server=5d4d7471f19e,37405,1733555609713 in 454 msec 2024-12-07T07:13:33,542 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened d60e3c4175109e97854d2b36c43a459f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62775222, jitterRate=-0.06457629799842834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,542 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:33,542 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for d60e3c4175109e97854d2b36c43a459f: Running coprocessor pre-open hook at 1733555613525Writing region info on filesystem at 1733555613525Initializing all the Stores at 1733555613527 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613527Cleaning up temporary data from old regions at 1733555613534 (+7 ms)Running coprocessor post-open hooks at 1733555613542 (+8 ms)Region opened successfully at 1733555613542 2024-12-07T07:13:33,544 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f., pid=50, masterSystemTime=1733555613256 2024-12-07T07:13:33,544 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:33,544 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:33,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c709bd9f88fc6dff6480583f422247fa, ASSIGN in 706 msec 2024-12-07T07:13:33,546 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:33,546 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:33,547 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:33,547 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 45e7ad8954e0a4d51f932a71d965426b, NAME => 'test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-07T07:13:33,547 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,547 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:33,547 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,548 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,549 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,550 INFO [StoreOpener-45e7ad8954e0a4d51f932a71d965426b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,553 INFO [StoreOpener-45e7ad8954e0a4d51f932a71d965426b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e7ad8954e0a4d51f932a71d965426b columnFamilyName family 2024-12-07T07:13:33,553 DEBUG [StoreOpener-45e7ad8954e0a4d51f932a71d965426b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:33,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure d60e3c4175109e97854d2b36c43a459f, server=5d4d7471f19e,34103,1733555609616 because future has completed 2024-12-07T07:13:33,555 INFO [StoreOpener-45e7ad8954e0a4d51f932a71d965426b-1 {}] regionserver.HStore(327): Store=45e7ad8954e0a4d51f932a71d965426b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:33,555 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:33,555 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,556 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,557 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,558 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,558 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure da5554a80a6c43e6ba1b1014d9f4023a, server=5d4d7471f19e,37405,1733555609713 because future has completed 2024-12-07T07:13:33,560 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,569 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=5d4d7471f19e,41097,1733555609760, table=test, region=2cfa28a210d48360878dfe1333fd8e26. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-07T07:13:33,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=28 2024-12-07T07:13:33,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure d60e3c4175109e97854d2b36c43a459f, server=5d4d7471f19e,34103,1733555609616 in 448 msec 2024-12-07T07:13:33,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d60e3c4175109e97854d2b36c43a459f, ASSIGN in 734 msec 2024-12-07T07:13:33,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-12-07T07:13:33,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure da5554a80a6c43e6ba1b1014d9f4023a, server=5d4d7471f19e,37405,1733555609713 in 511 msec 2024-12-07T07:13:33,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=da5554a80a6c43e6ba1b1014d9f4023a, ASSIGN in 740 msec 2024-12-07T07:13:33,579 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cf320858c98aa00078ada211b555e73e, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf320858c98aa00078ada211b555e73e, server=5d4d7471f19e,34103,1733555609616 because future has completed 2024-12-07T07:13:33,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=8 2024-12-07T07:13:33,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 764f719c1b7c5872a4cc2b9d21f2dc24, server=5d4d7471f19e,37405,1733555609713 in 484 msec 2024-12-07T07:13:33,585 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/45e7ad8954e0a4d51f932a71d965426b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T07:13:33,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=764f719c1b7c5872a4cc2b9d21f2dc24, ASSIGN in 748 msec 2024-12-07T07:13:33,586 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 45e7ad8954e0a4d51f932a71d965426b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71859309, jitterRate=0.07078714668750763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:33,586 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:33,586 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 45e7ad8954e0a4d51f932a71d965426b: Running coprocessor pre-open hook at 1733555613548Writing region info on filesystem at 1733555613548Initializing all the Stores at 1733555613549 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555613549Cleaning up temporary data from old regions at 1733555613558 (+9 ms)Running coprocessor post-open hooks at 1733555613586 (+28 ms)Region opened successfully at 1733555613586 2024-12-07T07:13:33,587 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b., pid=52, masterSystemTime=1733555613256 2024-12-07T07:13:33,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=7 2024-12-07T07:13:33,590 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:33,590 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:33,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure cf320858c98aa00078ada211b555e73e, server=5d4d7471f19e,34103,1733555609616 in 450 msec 2024-12-07T07:13:33,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cf320858c98aa00078ada211b555e73e, ASSIGN in 755 msec 2024-12-07T07:13:33,595 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, openSeqNum=2, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:33,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e7ad8954e0a4d51f932a71d965426b, server=5d4d7471f19e,34103,1733555609616 because future has completed 2024-12-07T07:13:33,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=17 2024-12-07T07:13:33,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c8344bda7e0c92708aba9a317b54627, server=5d4d7471f19e,37405,1733555609713 in 522 msec 2024-12-07T07:13:33,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4c8344bda7e0c92708aba9a317b54627, ASSIGN in 766 msec 2024-12-07T07:13:33,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=18 2024-12-07T07:13:33,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 45e7ad8954e0a4d51f932a71d965426b, server=5d4d7471f19e,34103,1733555609616 in 478 msec 2024-12-07T07:13:33,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=45e7ad8954e0a4d51f932a71d965426b, ASSIGN in 773 msec 2024-12-07T07:13:33,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=13 2024-12-07T07:13:33,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 2cfa28a210d48360878dfe1333fd8e26, server=5d4d7471f19e,41097,1733555609760 in 584 msec 2024-12-07T07:13:33,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN in 779 msec 2024-12-07T07:13:33,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=6 2024-12-07T07:13:33,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 58e2442015eb81a1979c8a3c79c8b4a4, server=5d4d7471f19e,37405,1733555609713 in 497 msec 2024-12-07T07:13:33,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=4 2024-12-07T07:13:33,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=58e2442015eb81a1979c8a3c79c8b4a4, ASSIGN in 786 msec 2024-12-07T07:13:33,627 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T07:13:33,628 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733555613628"}]},"ts":"1733555613628"} 2024-12-07T07:13:33,631 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-07T07:13:33,633 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T07:13:33,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5150 sec 2024-12-07T07:13:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T07:13:34,290 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-07T07:13:34,290 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-07T07:13:34,292 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T07:13:34,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-07T07:13:34,315 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T07:13:34,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-07T07:13:34,316 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T07:13:34,320 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-07T07:13:34,341 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-07T07:13:34,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T07:13:34,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T07:13:34,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T07:13:34,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T07:13:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cf320858c98aa00078ada211b555e73e changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6aadffb1e0d6160805fffee563de94ca changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5101f15d587bc41cd49e68708e4a5ed2 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 51e3bd49877ea2b244de43a10f4eaa37 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 45e7ad8954e0a4d51f932a71d965426b changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 37e2dfda6af2ae02ec7c14d6781d4503 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ffbaf4a5440811c93c49fb27af711a24 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5369f3b9703dae7de70d6556989c74a8 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d60e3c4175109e97854d2b36c43a459f changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 58e2442015eb81a1979c8a3c79c8b4a4 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 764f719c1b7c5872a4cc2b9d21f2dc24 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region caa1b434450bd250c5e2792eafa50df0 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a24f56f7916756473277b23c11f5ce84 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4c8344bda7e0c92708aba9a317b54627 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c709bd9f88fc6dff6480583f422247fa changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region da5554a80a6c43e6ba1b1014d9f4023a changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 99de37e290b4841263ba62805b625d97 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region abe9b4a81ffb7c674120d136c6ab6231 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ff1807b6b64d6c4ca2c6c6d7a2cafbd3 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region acd8e0c860fdbce58ce803c15220afb0 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2cfa28a210d48360878dfe1333fd8e26 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fe956aa50065b9ae6300ba6efb3d715d changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c1e55fd1b7105eca2dcbd1d5897f7f5a changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ab678da1dd813ac72686a84c4f22fdc0 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9d2616a94cd0a6eca969a2466add4db9 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d07cc4fd0d69fc55b90b071c1b0400c7 changed from -1.0 to 0.0, refreshing cache 2024-12-07T07:13:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-07T07:13:34,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-07T07:13:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-07T07:13:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(204): Hosts are {5d4d7471f19e=0} racks are {/default-rack=0} 2024-12-07T07:13:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-12-07T07:13:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-12-07T07:13:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-12-07T07:13:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T07:13:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T07:13:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-07T07:13:34,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T07:13:34,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T07:13:34,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-07T07:13:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-07T07:13:34,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-07T07:13:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-07T07:13:34,428 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T07:13:34,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-12-07T07:13:34,457 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=380, OpenFileDescriptor=637, MaxFileDescriptor=1048576, SystemLoadAverage=219, ProcessCount=11, AvailableMemoryMB=7049 2024-12-07T07:13:34,460 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-07T07:13:34,461 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-07T07:13:34,462 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:34,463 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 5d4d7471f19e,41097,1733555609760: Simulated kill ***** 2024-12-07T07:13:34,463 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-07T07:13:34,465 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-07T07:13:34,468 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-07T07:13:34,469 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-07T07:13:34,469 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-07T07:13:34,469 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 303291520 }, "NonHeapMemoryUsage": { "committed": 151781376, "init": 7667712, "max": -1, "used": 149421800 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "5d4d7471f19e" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "5d4d7471f19e" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "5d4d7471f19e" } ] } 2024-12-07T07:13:34,473 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46437 {}] master.MasterRpcServices(700): 5d4d7471f19e,41097,1733555609760 reported a fatal error: ***** ABORTING region server 5d4d7471f19e,41097,1733555609760: Simulated kill ***** 2024-12-07T07:13:34,475 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5d4d7471f19e,41097,1733555609760' ***** 2024-12-07T07:13:34,475 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-07T07:13:34,476 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T07:13:34,476 INFO [RS:2;5d4d7471f19e:41097 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-07T07:13:34,476 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T07:13:34,476 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T07:13:34,476 INFO [RS:2;5d4d7471f19e:41097 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-07T07:13:34,476 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T07:13:34,477 INFO [RS:2;5d4d7471f19e:41097 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;5d4d7471f19e:41097. 2024-12-07T07:13:34,477 DEBUG [RS:2;5d4d7471f19e:41097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T07:13:34,481 DEBUG [RS:2;5d4d7471f19e:41097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T07:13:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741836_1012 (size=32114) 2024-12-07T07:13:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741836_1012 (size=32114) 2024-12-07T07:13:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741836_1012 (size=32114) 2024-12-07T07:13:34,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741835_1011 (size=2232) 2024-12-07T07:13:34,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741835_1011 (size=2232) 2024-12-07T07:13:34,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741835_1011 (size=2232) 2024-12-07T07:13:34,491 DEBUG [RS:2;5d4d7471f19e:41097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T07:13:34,491 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T07:13:34,492 INFO [RS:2;5d4d7471f19e:41097 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41097 2024-12-07T07:13:34,504 INFO [regionserver/5d4d7471f19e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T07:13:34,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5d4d7471f19e,41097,1733555609760 2024-12-07T07:13:34,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T07:13:34,619 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T07:13:34,629 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5d4d7471f19e,41097,1733555609760] 2024-12-07T07:13:34,639 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5d4d7471f19e,41097,1733555609760 already deleted, retry=false 2024-12-07T07:13:34,639 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 5d4d7471f19e,41097,1733555609760 on 5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:34,643 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:34,648 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:34,648 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 5d4d7471f19e,41097,1733555609760 (carryingMeta=true) 5d4d7471f19e,41097,1733555609760/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@593b2405[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-07T07:13:34,649 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true, isMeta: true 2024-12-07T07:13:34,651 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting 2024-12-07T07:13:34,653 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5d4d7471f19e,41097,1733555609760 WAL count=1, meta=true 2024-12-07T07:13:34,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta}] 2024-12-07T07:13:34,660 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:34,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:34,678 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T07:13:34,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T07:13:34,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41097-0x101aea355730003, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T07:13:34,730 INFO [RS:2;5d4d7471f19e:41097 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T07:13:34,730 INFO [RS:2;5d4d7471f19e:41097 {}] regionserver.HRegionServer(1031): Exiting; stopping=5d4d7471f19e,41097,1733555609760; zookeeper connection closed. 2024-12-07T07:13:34,730 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@58831182 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@58831182 2024-12-07T07:13:34,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34103 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-12-07T07:13:34,859 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, size=31.4 K (32114bytes) 2024-12-07T07:13:34,860 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-07T07:13:34,860 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-07T07:13:34,860 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta 2024-12-07T07:13:34,868 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta after 5ms 2024-12-07T07:13:34,876 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta took 18ms 2024-12-07T07:13:34,889 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-07T07:13:34,895 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta.temp 2024-12-07T07:13:34,898 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta so closing down 2024-12-07T07:13:34,898 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T07:13:34,902 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta.temp 2024-12-07T07:13:34,906 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T07:13:34,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741864_1040 (size=31873) 2024-12-07T07:13:34,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741864_1040 (size=31873) 2024-12-07T07:13:34,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741864_1040 (size=31873) 2024-12-07T07:13:34,935 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-12-07T07:13:34,939 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta.temp to hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-07T07:13:34,944 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 61 ms; skipped=1; WAL=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-12-07T07:13:34,944 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, journal: Splitting hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, size=31.4 K (32114bytes) at 1733555614859Finishing writing output for hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta so closing down at 1733555614898 (+39 ms)Creating recovered edits writer path=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta.temp at 1733555614902 (+4 ms)3 split writer threads finished at 1733555614906 (+4 ms)Closed recovered edits writer path=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1733555614935 (+29 ms)Rename recovered edits hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta.temp to hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1733555614939 (+4 ms)Processed 58 edits across 1 Regions in 61 ms; skipped=1; WAL=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1733555614944 (+5 ms) 2024-12-07T07:13:34,945 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta 2024-12-07T07:13:34,947 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-07T07:13:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-07T07:13:34,962 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta to hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/oldWALs 2024-12-07T07:13:34,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-07T07:13:34,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 in 300 msec 2024-12-07T07:13:34,974 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:34,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-07T07:13:34,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 in 321 msec 2024-12-07T07:13:34,998 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting dir is empty, no logs to split. 2024-12-07T07:13:34,998 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5d4d7471f19e,41097,1733555609760 WAL count=0, meta=true 2024-12-07T07:13:34,998 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 5d4d7471f19e,41097,1733555609760 WAL splitting is done? wals=0, meta=true 2024-12-07T07:13:35,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T07:13:35,006 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:35,008 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T07:13:35,101 INFO [regionserver/5d4d7471f19e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-07T07:13:35,101 INFO [regionserver/5d4d7471f19e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-07T07:13:35,159 DEBUG [5d4d7471f19e:46437 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-07T07:13:35,159 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(204): Hosts are {5d4d7471f19e=0} racks are {/default-rack=0} 2024-12-07T07:13:35,159 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T07:13:35,159 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T07:13:35,159 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T07:13:35,159 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T07:13:35,159 INFO [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T07:13:35,159 INFO [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T07:13:35,159 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T07:13:35,160 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:35,161 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPENING 2024-12-07T07:13:35,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:35,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:35,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:35,208 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:35,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:35,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:35,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:35,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:35,366 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T07:13:35,366 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T07:13:35,367 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T07:13:35,369 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5d4d7471f19e%2C34103%2C1733555609616.meta, suffix=.meta, logDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,34103,1733555609616, archiveDir=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/oldWALs, maxLogs=32 2024-12-07T07:13:35,381 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,34103,1733555609616/5d4d7471f19e%2C34103%2C1733555609616.meta.1733555615370.meta, exclude list is [], retry=0 2024-12-07T07:13:35,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41211,DS-db1aad08-c885-4b21-a48f-8bcb9ef25c82,DISK] 2024-12-07T07:13:35,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44427,DS-0022ef86-b620-4ac8-b9f6-b3b559d4f8a4,DISK] 2024-12-07T07:13:35,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38133,DS-77296bb1-6178-4837-b9f0-b2f5cd04c8b3,DISK] 2024-12-07T07:13:35,387 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,34103,1733555609616/5d4d7471f19e%2C34103%2C1733555609616.meta.1733555615370.meta 2024-12-07T07:13:35,388 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37139:37139),(127.0.0.1/127.0.0.1:42707:42707),(127.0.0.1/127.0.0.1:35525:35525)] 2024-12-07T07:13:35,388 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T07:13:35,388 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T07:13:35,388 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T07:13:35,388 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T07:13:35,389 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T07:13:35,389 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:35,389 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T07:13:35,389 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T07:13:35,391 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T07:13:35,392 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T07:13:35,392 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:35,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:35,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T07:13:35,394 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T07:13:35,394 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:35,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:35,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T07:13:35,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T07:13:35,396 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:35,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:35,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T07:13:35,397 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T07:13:35,397 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:35,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T07:13:35,398 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T07:13:35,399 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740 2024-12-07T07:13:35,401 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740 2024-12-07T07:13:35,402 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-07T07:13:35,417 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-07T07:13:35,420 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-12-07T07:13:35,493 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/info/0f1898fdecb94421b0ad3e08f65384d8 is 138, key is test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4./info:regioninfo/1733555613287/Put/seqid=0 2024-12-07T07:13:35,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741866_1042 (size=34924) 2024-12-07T07:13:35,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741866_1042 (size=34924) 2024-12-07T07:13:35,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741866_1042 (size=34924) 2024-12-07T07:13:35,506 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/info/0f1898fdecb94421b0ad3e08f65384d8 2024-12-07T07:13:35,570 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/ns/1c3cc69dc29f4dd2b86ac4bf4ccc319a is 43, key is default/ns:d/1733555611909/Put/seqid=0 2024-12-07T07:13:35,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741867_1043 (size=5153) 2024-12-07T07:13:35,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741867_1043 (size=5153) 2024-12-07T07:13:35,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741867_1043 (size=5153) 2024-12-07T07:13:35,579 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/ns/1c3cc69dc29f4dd2b86ac4bf4ccc319a 2024-12-07T07:13:35,619 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/table/034dc4f5f1554f71b27f0c0ef5c222dd is 40, key is test/table:state/1733555613628/Put/seqid=0 2024-12-07T07:13:35,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741868_1044 (size=5165) 2024-12-07T07:13:35,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741868_1044 (size=5165) 2024-12-07T07:13:35,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741868_1044 (size=5165) 2024-12-07T07:13:35,626 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/table/034dc4f5f1554f71b27f0c0ef5c222dd 2024-12-07T07:13:35,635 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/info/0f1898fdecb94421b0ad3e08f65384d8 as hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/info/0f1898fdecb94421b0ad3e08f65384d8 2024-12-07T07:13:35,644 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/info/0f1898fdecb94421b0ad3e08f65384d8, entries=260, sequenceid=60, filesize=34.1 K 2024-12-07T07:13:35,647 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/ns/1c3cc69dc29f4dd2b86ac4bf4ccc319a as hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/ns/1c3cc69dc29f4dd2b86ac4bf4ccc319a 2024-12-07T07:13:35,656 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/ns/1c3cc69dc29f4dd2b86ac4bf4ccc319a, entries=2, sequenceid=60, filesize=5.0 K 2024-12-07T07:13:35,657 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/table/034dc4f5f1554f71b27f0c0ef5c222dd as hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/table/034dc4f5f1554f71b27f0c0ef5c222dd 2024-12-07T07:13:35,667 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/table/034dc4f5f1554f71b27f0c0ef5c222dd, entries=2, sequenceid=60, filesize=5.0 K 2024-12-07T07:13:35,672 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 251ms, sequenceid=60, compaction requested=false; wal=null 2024-12-07T07:13:35,674 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-07T07:13:35,678 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-07T07:13:35,680 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T07:13:35,680 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T07:13:35,681 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T07:13:35,683 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T07:13:35,686 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-12-07T07:13:35,687 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73547301, jitterRate=0.09594018757343292}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T07:13:35,687 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T07:13:35,687 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733555615389Writing region info on filesystem at 1733555615389Initializing all the Stores at 1733555615390 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733555615390Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733555615391 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555615391Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733555615391Obtaining lock to block concurrent updates at 1733555615420 (+29 ms)Preparing flush snapshotting stores in 1588230740 at 1733555615420Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1733555615423 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733555615423Flushing 1588230740/info: creating writer at 1733555615425 (+2 ms)Flushing 1588230740/info: appending metadata at 1733555615484 (+59 ms)Flushing 1588230740/info: closing flushed file at 1733555615486 (+2 ms)Flushing 1588230740/ns: creating writer at 1733555615551 (+65 ms)Flushing 1588230740/ns: appending metadata at 1733555615569 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733555615569Flushing 1588230740/table: creating writer at 1733555615587 (+18 ms)Flushing 1588230740/table: appending metadata at 1733555615618 (+31 ms)Flushing 1588230740/table: closing flushed file at 1733555615618Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11becd36: reopening flushed file at 1733555615634 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40cab131: reopening flushed file at 1733555615646 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dd5c6bc: reopening flushed file at 1733555615656 (+10 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 251ms, sequenceid=60, compaction requested=false; wal=null at 1733555615673 (+17 ms)Cleaning up temporary data from old regions at 1733555615680 (+7 ms)Running coprocessor post-open hooks at 1733555615687 (+7 ms)Region opened successfully at 1733555615687 2024-12-07T07:13:35,689 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1733555615361 2024-12-07T07:13:35,693 DEBUG [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T07:13:35,693 INFO [RS_OPEN_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T07:13:35,693 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:35,695 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:35,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:35,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:35,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:35,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:35,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:35,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:35,745 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:35,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-07T07:13:35,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 in 537 msec 2024-12-07T07:13:35,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-12-07T07:13:35,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 746 msec 2024-12-07T07:13:35,751 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 5d4d7471f19e,41097,1733555609760 had 9 regions 2024-12-07T07:13:35,753 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true, isMeta: false 2024-12-07T07:13:35,755 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5d4d7471f19e,41097,1733555609760 WAL count=1, meta=false 2024-12-07T07:13:35,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238}] 2024-12-07T07:13:35,758 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:35,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:35,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34103 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-12-07T07:13:35,932 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238, size=2.2 K (2232bytes) 2024-12-07T07:13:35,932 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238 2024-12-07T07:13:35,932 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238 after 0ms 2024-12-07T07:13:35,936 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238 took 5ms 2024-12-07T07:13:35,939 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for acd8e0c860fdbce58ce803c15220afb0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T07:13:35,941 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d07cc4fd0d69fc55b90b071c1b0400c7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T07:13:35,943 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c1e55fd1b7105eca2dcbd1d5897f7f5a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T07:13:35,945 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ff1807b6b64d6c4ca2c6c6d7a2cafbd3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T07:13:35,947 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ab678da1dd813ac72686a84c4f22fdc0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T07:13:35,949 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2cfa28a210d48360878dfe1333fd8e26: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T07:13:35,950 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 9d2616a94cd0a6eca969a2466add4db9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T07:13:35,952 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for fe956aa50065b9ae6300ba6efb3d715d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-07T07:13:35,952 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238 so closing down 2024-12-07T07:13:35,952 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T07:13:35,953 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T07:13:35,953 INFO [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 17 ms; skipped=8; WAL=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238, size=2.2 K, length=2232, corrupted=false, cancelled=false 2024-12-07T07:13:35,953 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238, journal: Splitting hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238, size=2.2 K (2232bytes) at 1733555615932Finishing writing output for hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238 so closing down at 1733555615952 (+20 ms)3 split writer threads finished at 1733555615953 (+1 ms)Processed 8 edits across 0 Regions in 17 ms; skipped=8; WAL=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238, size=2.2 K, length=2232, corrupted=false, cancelled=false at 1733555615953 2024-12-07T07:13:35,953 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238 2024-12-07T07:13:35,953 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5d4d7471f19e:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-07T07:13:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46437 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-07T07:13:35,959 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting/5d4d7471f19e%2C41097%2C1733555609760.1733555611238 to hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/oldWALs 2024-12-07T07:13:35,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-07T07:13:35,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 in 200 msec 2024-12-07T07:13:35,962 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:35,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-12-07T07:13:35,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 in 208 msec 2024-12-07T07:13:35,967 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/WALs/5d4d7471f19e,41097,1733555609760-splitting dir is empty, no logs to split. 2024-12-07T07:13:35,967 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5d4d7471f19e,41097,1733555609760 WAL count=0, meta=false 2024-12-07T07:13:35,967 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 5d4d7471f19e,41097,1733555609760 WAL splitting is done? wals=0, meta=false 2024-12-07T07:13:35,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN}] 2024-12-07T07:13:35,973 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:35,974 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T07:13:35,975 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:35,976 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T07:13:35,977 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:35,978 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T07:13:35,978 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:35,979 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T07:13:35,980 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:35,981 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T07:13:35,982 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:35,983 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T07:13:35,984 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:35,984 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T07:13:35,985 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:35,986 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T07:13:36,125 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(204): Hosts are {5d4d7471f19e=0} racks are {/default-rack=0} 2024-12-07T07:13:36,125 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T07:13:36,125 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T07:13:36,125 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T07:13:36,125 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T07:13:36,125 INFO [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T07:13:36,125 INFO [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T07:13:36,125 DEBUG [5d4d7471f19e:46437 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T07:13:36,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,127 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, regionLocation=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,127 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 5d4d7471f19e:41097 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T07:13:36,127 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,128 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,129 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1, error=java.net.ConnectException: Call to address=5d4d7471f19e:41097 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 2024-12-07T07:13:36,129 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, regionLocation=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-07T07:13:36,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1 from cache 2024-12-07T07:13:36,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=5d4d7471f19e:41097 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 2024-12-07T07:13:36,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 5d4d7471f19e:41097 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 2024-12-07T07:13:36,131 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 5d4d7471f19e:41097 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T07:13:36,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=5d4d7471f19e:41097 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 2024-12-07T07:13:36,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=5d4d7471f19e:41097 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 2024-12-07T07:13:36,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=5d4d7471f19e:41097 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 2024-12-07T07:13:36,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=5d4d7471f19e:41097 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 2024-12-07T07:13:36,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=5d4d7471f19e:41097 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 2024-12-07T07:13:36,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,41097,1733555609760, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=5d4d7471f19e:41097 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 2024-12-07T07:13:36,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 5d4d7471f19e:41097 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 5d4d7471f19e/172.17.0.2:41097 2024-12-07T07:13:36,237 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T07:13:36,237 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5d4d7471f19e,34103,1733555609616, seqNum=-1] 2024-12-07T07:13:36,237 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T07:13:36,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38559, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T07:13:36,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN because future has completed 2024-12-07T07:13:36,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:36,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN because future has completed 2024-12-07T07:13:36,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN because future has completed 2024-12-07T07:13:36,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN because future has completed 2024-12-07T07:13:36,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN because future has completed 2024-12-07T07:13:36,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:36,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN because future has completed 2024-12-07T07:13:36,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN because future has completed 2024-12-07T07:13:36,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN because future has completed 2024-12-07T07:13:36,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:36,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:36,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713}] 2024-12-07T07:13:36,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:36,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfa28a210d48360878dfe1333fd8e26, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:36,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616}] 2024-12-07T07:13:36,404 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:36,404 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => c1e55fd1b7105eca2dcbd1d5897f7f5a, NAME => 'test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-07T07:13:36,404 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,404 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:36,405 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,405 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,406 INFO [StoreOpener-c1e55fd1b7105eca2dcbd1d5897f7f5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,407 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:36,407 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => ab678da1dd813ac72686a84c4f22fdc0, NAME => 'test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-07T07:13:36,407 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,407 INFO [StoreOpener-c1e55fd1b7105eca2dcbd1d5897f7f5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1e55fd1b7105eca2dcbd1d5897f7f5a columnFamilyName family 2024-12-07T07:13:36,408 DEBUG [StoreOpener-c1e55fd1b7105eca2dcbd1d5897f7f5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:36,408 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:36,408 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,408 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,408 INFO [StoreOpener-c1e55fd1b7105eca2dcbd1d5897f7f5a-1 {}] regionserver.HStore(327): Store=c1e55fd1b7105eca2dcbd1d5897f7f5a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:36,408 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,409 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,409 INFO [StoreOpener-ab678da1dd813ac72686a84c4f22fdc0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,410 INFO [StoreOpener-ab678da1dd813ac72686a84c4f22fdc0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab678da1dd813ac72686a84c4f22fdc0 columnFamilyName family 2024-12-07T07:13:36,410 DEBUG [StoreOpener-ab678da1dd813ac72686a84c4f22fdc0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:36,411 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,411 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,411 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,411 INFO [StoreOpener-ab678da1dd813ac72686a84c4f22fdc0-1 {}] regionserver.HStore(327): Store=ab678da1dd813ac72686a84c4f22fdc0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:36,411 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,412 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,414 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,414 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,414 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,414 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,415 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened c1e55fd1b7105eca2dcbd1d5897f7f5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61265984, jitterRate=-0.0870656967163086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:36,415 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:36,415 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for c1e55fd1b7105eca2dcbd1d5897f7f5a: Running coprocessor pre-open hook at 1733555616405Writing region info on filesystem at 1733555616405Initializing all the Stores at 1733555616406 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555616406Cleaning up temporary data from old regions at 1733555616411 (+5 ms)Running coprocessor post-open hooks at 1733555616415 (+4 ms)Region opened successfully at 1733555616415 2024-12-07T07:13:36,416 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a., pid=79, masterSystemTime=1733555616394 2024-12-07T07:13:36,417 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,418 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened ab678da1dd813ac72686a84c4f22fdc0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71227593, jitterRate=0.06137384474277496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:36,418 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:36,418 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for ab678da1dd813ac72686a84c4f22fdc0: Running coprocessor pre-open hook at 1733555616408Writing region info on filesystem at 1733555616408Initializing all the Stores at 1733555616409 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555616409Cleaning up temporary data from old regions at 1733555616414 (+5 ms)Running coprocessor post-open hooks at 1733555616418 (+4 ms)Region opened successfully at 1733555616418 2024-12-07T07:13:36,419 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:36,419 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:36,419 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:36,419 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d2616a94cd0a6eca969a2466add4db9, NAME => 'test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-07T07:13:36,419 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,419 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:36,419 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0., pid=76, masterSystemTime=1733555616400 2024-12-07T07:13:36,420 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,420 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,421 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-07T07:13:36,422 INFO [StoreOpener-9d2616a94cd0a6eca969a2466add4db9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,422 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,422 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:36,422 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:36,422 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:36,423 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => d07cc4fd0d69fc55b90b071c1b0400c7, NAME => 'test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-07T07:13:36,423 INFO [StoreOpener-9d2616a94cd0a6eca969a2466add4db9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d2616a94cd0a6eca969a2466add4db9 columnFamilyName family 2024-12-07T07:13:36,423 DEBUG [StoreOpener-9d2616a94cd0a6eca969a2466add4db9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:36,422 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,423 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,423 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:36,423 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,423 INFO [StoreOpener-9d2616a94cd0a6eca969a2466add4db9-1 {}] regionserver.HStore(327): Store=9d2616a94cd0a6eca969a2466add4db9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:36,423 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,424 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,424 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,426 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,427 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,427 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,428 INFO [StoreOpener-d07cc4fd0d69fc55b90b071c1b0400c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,429 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,429 INFO [StoreOpener-d07cc4fd0d69fc55b90b071c1b0400c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d07cc4fd0d69fc55b90b071c1b0400c7 columnFamilyName family 2024-12-07T07:13:36,429 DEBUG [StoreOpener-d07cc4fd0d69fc55b90b071c1b0400c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:36,430 INFO [StoreOpener-d07cc4fd0d69fc55b90b071c1b0400c7-1 {}] regionserver.HStore(327): Store=d07cc4fd0d69fc55b90b071c1b0400c7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:36,430 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,430 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 9d2616a94cd0a6eca969a2466add4db9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59952141, jitterRate=-0.10664348304271698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:36,430 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:36,431 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 9d2616a94cd0a6eca969a2466add4db9: Running coprocessor pre-open hook at 1733555616420Writing region info on filesystem at 1733555616420Initializing all the Stores at 1733555616421 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555616421Cleaning up temporary data from old regions at 1733555616427 (+6 ms)Running coprocessor post-open hooks at 1733555616431 (+4 ms)Region opened successfully at 1733555616431 2024-12-07T07:13:36,431 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9., pid=72, masterSystemTime=1733555616394 2024-12-07T07:13:36,431 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,433 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,432 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-12-07T07:13:36,434 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,434 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,434 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:36,434 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:36,434 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:36,434 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,434 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,434 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => ff1807b6b64d6c4ca2c6c6d7a2cafbd3, NAME => 'test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-07T07:13:36,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,435 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,435 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:36,435 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,435 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,435 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,436 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,436 INFO [StoreOpener-ff1807b6b64d6c4ca2c6c6d7a2cafbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,437 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened d07cc4fd0d69fc55b90b071c1b0400c7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61719773, jitterRate=-0.08030371367931366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:36,437 INFO [StoreOpener-ff1807b6b64d6c4ca2c6c6d7a2cafbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff1807b6b64d6c4ca2c6c6d7a2cafbd3 columnFamilyName family 2024-12-07T07:13:36,437 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:36,437 DEBUG [StoreOpener-ff1807b6b64d6c4ca2c6c6d7a2cafbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:36,438 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for d07cc4fd0d69fc55b90b071c1b0400c7: Running coprocessor pre-open hook at 1733555616424Writing region info on filesystem at 1733555616424Initializing all the Stores at 1733555616425 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555616425Cleaning up temporary data from old regions at 1733555616434 (+9 ms)Running coprocessor post-open hooks at 1733555616437 (+3 ms)Region opened successfully at 1733555616437 2024-12-07T07:13:36,438 INFO [StoreOpener-ff1807b6b64d6c4ca2c6c6d7a2cafbd3-1 {}] regionserver.HStore(327): Store=ff1807b6b64d6c4ca2c6c6d7a2cafbd3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:36,438 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,439 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7., pid=73, masterSystemTime=1733555616400 2024-12-07T07:13:36,439 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,440 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,441 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,441 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,442 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:36,442 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:36,442 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:36,442 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => fe956aa50065b9ae6300ba6efb3d715d, NAME => 'test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-07T07:13:36,443 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,443 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:36,443 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,443 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,443 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,444 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened ff1807b6b64d6c4ca2c6c6d7a2cafbd3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59361147, jitterRate=-0.11544997990131378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:36,444 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:36,444 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for ff1807b6b64d6c4ca2c6c6d7a2cafbd3: Running coprocessor pre-open hook at 1733555616435Writing region info on filesystem at 1733555616435Initializing all the Stores at 1733555616436 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555616436Cleaning up temporary data from old regions at 1733555616441 (+5 ms)Running coprocessor post-open hooks at 1733555616444 (+3 ms)Region opened successfully at 1733555616444 2024-12-07T07:13:36,444 INFO [StoreOpener-fe956aa50065b9ae6300ba6efb3d715d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,445 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3., pid=77, masterSystemTime=1733555616394 2024-12-07T07:13:36,446 INFO [StoreOpener-fe956aa50065b9ae6300ba6efb3d715d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe956aa50065b9ae6300ba6efb3d715d columnFamilyName family 2024-12-07T07:13:36,446 DEBUG [StoreOpener-fe956aa50065b9ae6300ba6efb3d715d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:36,446 INFO [StoreOpener-fe956aa50065b9ae6300ba6efb3d715d-1 {}] regionserver.HStore(327): Store=fe956aa50065b9ae6300ba6efb3d715d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:36,446 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,447 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,448 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:36,448 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:36,448 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:36,448 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 2cfa28a210d48360878dfe1333fd8e26, NAME => 'test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-07T07:13:36,448 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,448 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:36,449 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,449 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,449 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,449 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,449 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,450 INFO [StoreOpener-2cfa28a210d48360878dfe1333fd8e26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,451 INFO [StoreOpener-2cfa28a210d48360878dfe1333fd8e26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cfa28a210d48360878dfe1333fd8e26 columnFamilyName family 2024-12-07T07:13:36,451 DEBUG [StoreOpener-2cfa28a210d48360878dfe1333fd8e26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:36,451 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,451 INFO [StoreOpener-2cfa28a210d48360878dfe1333fd8e26-1 {}] regionserver.HStore(327): Store=2cfa28a210d48360878dfe1333fd8e26/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:36,451 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,452 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,452 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened fe956aa50065b9ae6300ba6efb3d715d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67764533, jitterRate=0.009770229458808899}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:36,452 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:36,452 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for fe956aa50065b9ae6300ba6efb3d715d: Running coprocessor pre-open hook at 1733555616443Writing region info on filesystem at 1733555616443Initializing all the Stores at 1733555616444 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555616444Cleaning up temporary data from old regions at 1733555616449 (+5 ms)Running coprocessor post-open hooks at 1733555616452 (+3 ms)Region opened successfully at 1733555616452 2024-12-07T07:13:36,453 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d., pid=75, masterSystemTime=1733555616400 2024-12-07T07:13:36,453 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,454 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,454 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,456 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:36,456 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:36,456 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:36,456 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => acd8e0c860fdbce58ce803c15220afb0, NAME => 'test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-07T07:13:36,456 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,456 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,456 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T07:13:36,457 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,457 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,457 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 2cfa28a210d48360878dfe1333fd8e26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71520469, jitterRate=0.06573803722858429}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:36,458 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:36,458 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 2cfa28a210d48360878dfe1333fd8e26: Running coprocessor pre-open hook at 1733555616449Writing region info on filesystem at 1733555616449Initializing all the Stores at 1733555616450 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555616450Cleaning up temporary data from old regions at 1733555616454 (+4 ms)Running coprocessor post-open hooks at 1733555616458 (+4 ms)Region opened successfully at 1733555616458 2024-12-07T07:13:36,458 INFO [StoreOpener-acd8e0c860fdbce58ce803c15220afb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,459 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26., pid=78, masterSystemTime=1733555616394 2024-12-07T07:13:36,459 INFO [StoreOpener-acd8e0c860fdbce58ce803c15220afb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acd8e0c860fdbce58ce803c15220afb0 columnFamilyName family 2024-12-07T07:13:36,459 DEBUG [StoreOpener-acd8e0c860fdbce58ce803c15220afb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T07:13:36,459 INFO [StoreOpener-acd8e0c860fdbce58ce803c15220afb0-1 {}] regionserver.HStore(327): Store=acd8e0c860fdbce58ce803c15220afb0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T07:13:36,460 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,461 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,461 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:36,461 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:36,462 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,462 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,462 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,464 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,465 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened acd8e0c860fdbce58ce803c15220afb0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68947245, jitterRate=0.027394011616706848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T07:13:36,465 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:36,465 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for acd8e0c860fdbce58ce803c15220afb0: Running coprocessor pre-open hook at 1733555616457Writing region info on filesystem at 1733555616457Initializing all the Stores at 1733555616458 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733555616458Cleaning up temporary data from old regions at 1733555616462 (+4 ms)Running coprocessor post-open hooks at 1733555616465 (+3 ms)Region opened successfully at 1733555616465 2024-12-07T07:13:36,466 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0., pid=74, masterSystemTime=1733555616400 2024-12-07T07:13:36,468 DEBUG [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:36,468 INFO [RS_OPEN_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:36,481 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,481 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,481 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,482 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,482 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,482 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,482 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,483 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,484 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,484 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,486 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,490 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,491 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,491 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,493 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfa28a210d48360878dfe1333fd8e26, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,498 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,498 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfa28a210d48360878dfe1333fd8e26, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,498 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-12-07T07:13:36,500 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,500 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,500 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,557 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,557 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,557 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,557 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,557 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,557 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,557 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=ab678da1dd813ac72686a84c4f22fdc0 to restore RIT 2024-12-07T07:13:36,557 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN to state=OFFLINE, location=null, table=test, region=9d2616a94cd0a6eca969a2466add4db9 to restore RIT 2024-12-07T07:13:36,558 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646558 2024-12-07T07:13:36,558 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,558 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,560 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,561 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,561 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfa28a210d48360878dfe1333fd8e26, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,561 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfa28a210d48360878dfe1333fd8e26, server=5d4d7471f19e,34103,1733555609616 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,599 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,599 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,599 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,599 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,599 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676599 2024-12-07T07:13:36,599 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736599 2024-12-07T07:13:36,599 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 41 msec 2024-12-07T07:13:36,600 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,600 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,600 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,600 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,600 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,600 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,600 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,601 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,602 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-07T07:13:36,607 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,608 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,609 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,609 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfa28a210d48360878dfe1333fd8e26, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,615 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,616 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,616 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,617 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,617 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,617 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,617 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,617 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,617 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,617 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,617 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,617 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,617 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,617 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfa28a210d48360878dfe1333fd8e26, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,617 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-12-07T07:13:36,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,619 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,619 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,629 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,629 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,629 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,630 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,630 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,630 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,630 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,630 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,630 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,630 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=ab678da1dd813ac72686a84c4f22fdc0 to restore RIT 2024-12-07T07:13:36,630 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN to state=OFFLINE, location=null, table=test, region=9d2616a94cd0a6eca969a2466add4db9 to restore RIT 2024-12-07T07:13:36,631 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646631 2024-12-07T07:13:36,631 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,631 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,634 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,634 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,634 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,34103,1733555609616 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,643 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,643 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676643 2024-12-07T07:13:36,643 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736643 2024-12-07T07:13:36,643 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-12-07T07:13:36,643 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,643 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,643 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,643 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,643 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,643 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,643 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,643 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,644 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,649 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,652 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff1807b6b64d6c4ca2c6c6d7a2cafbd3, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,652 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-07T07:13:36,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,653 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,653 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=ab678da1dd813ac72686a84c4f22fdc0 to restore RIT 2024-12-07T07:13:36,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN to state=OFFLINE, location=null, table=test, region=9d2616a94cd0a6eca969a2466add4db9 to restore RIT 2024-12-07T07:13:36,662 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646662 2024-12-07T07:13:36,662 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,662 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,663 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,664 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,664 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,664 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,673 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,674 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676673 2024-12-07T07:13:36,674 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736674 2024-12-07T07:13:36,674 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-12-07T07:13:36,674 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,674 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,674 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,674 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,674 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,675 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,676 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,680 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,687 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,687 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,687 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,687 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,687 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,687 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,687 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,688 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,688 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,688 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,688 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,688 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,688 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,688 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,688 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab678da1dd813ac72686a84c4f22fdc0, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,688 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-12-07T07:13:36,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,689 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,689 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,691 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,693 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,703 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,703 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,703 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,703 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,703 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,704 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,704 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,704 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,704 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,704 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=ab678da1dd813ac72686a84c4f22fdc0 to restore RIT 2024-12-07T07:13:36,704 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN to state=OFFLINE, location=null, table=test, region=9d2616a94cd0a6eca969a2466add4db9 to restore RIT 2024-12-07T07:13:36,704 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646704 2024-12-07T07:13:36,704 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,704 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,705 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,706 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,706 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,707 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,715 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,716 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,716 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,716 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,716 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,718 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,718 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676718 2024-12-07T07:13:36,718 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736718 2024-12-07T07:13:36,718 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-12-07T07:13:36,718 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,718 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,718 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,718 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,719 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,719 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,719 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,719 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,720 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,722 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe956aa50065b9ae6300ba6efb3d715d, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,728 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-07T07:13:36,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,729 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,729 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,741 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,741 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,741 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,741 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,742 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,742 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,742 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,742 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,742 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,742 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=ab678da1dd813ac72686a84c4f22fdc0 to restore RIT 2024-12-07T07:13:36,742 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN to state=OFFLINE, location=null, table=test, region=9d2616a94cd0a6eca969a2466add4db9 to restore RIT 2024-12-07T07:13:36,742 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646742 2024-12-07T07:13:36,742 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,742 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,744 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,746 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,746 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,746 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,754 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,754 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676754 2024-12-07T07:13:36,754 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736754 2024-12-07T07:13:36,754 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-07T07:13:36,754 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,754 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,754 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,755 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,755 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,755 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,755 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,755 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,756 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,760 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,763 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,764 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,765 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,765 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure acd8e0c860fdbce58ce803c15220afb0, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,766 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-07T07:13:36,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,767 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,767 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,776 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,776 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,776 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=ab678da1dd813ac72686a84c4f22fdc0 to restore RIT 2024-12-07T07:13:36,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN to state=OFFLINE, location=null, table=test, region=9d2616a94cd0a6eca969a2466add4db9 to restore RIT 2024-12-07T07:13:36,777 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646777 2024-12-07T07:13:36,778 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,778 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,779 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,780 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,786 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,786 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676786 2024-12-07T07:13:36,786 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736786 2024-12-07T07:13:36,786 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-07T07:13:36,786 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,786 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,787 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,787 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,787 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,787 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,788 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,789 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,790 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,793 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,793 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure d07cc4fd0d69fc55b90b071c1b0400c7, server=5d4d7471f19e,37405,1733555609713 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-07T07:13:36,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,794 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,794 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,804 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,805 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,805 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,805 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,805 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,805 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,805 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,805 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,805 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=ab678da1dd813ac72686a84c4f22fdc0 to restore RIT 2024-12-07T07:13:36,805 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN to state=OFFLINE, location=null, table=test, region=9d2616a94cd0a6eca969a2466add4db9 to restore RIT 2024-12-07T07:13:36,806 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646806 2024-12-07T07:13:36,806 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,806 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,807 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,808 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,809 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,809 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,816 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,816 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676816 2024-12-07T07:13:36,816 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736816 2024-12-07T07:13:36,816 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-07T07:13:36,817 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,817 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,817 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,817 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,817 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,817 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,818 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,819 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,821 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,823 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,823 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,823 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,823 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,823 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,823 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,823 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,823 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d2616a94cd0a6eca969a2466add4db9, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,824 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-07T07:13:36,824 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,824 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,824 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=ab678da1dd813ac72686a84c4f22fdc0 to restore RIT 2024-12-07T07:13:36,836 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN to state=OFFLINE, location=null, table=test, region=9d2616a94cd0a6eca969a2466add4db9 to restore RIT 2024-12-07T07:13:36,837 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646837 2024-12-07T07:13:36,837 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,837 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,838 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,839 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,839 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,839 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,846 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,846 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676846 2024-12-07T07:13:36,846 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736846 2024-12-07T07:13:36,847 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-07T07:13:36,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,847 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,847 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,847 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,847 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,848 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN 2024-12-07T07:13:36,853 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-07T07:13:36,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,854 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,854 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,860 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,860 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,860 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,861 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,861 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,861 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,861 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,861 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,861 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,861 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=ab678da1dd813ac72686a84c4f22fdc0 to restore RIT 2024-12-07T07:13:36,861 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9d2616a94cd0a6eca969a2466add4db9, ASSIGN to state=OFFLINE, location=null, table=test, region=9d2616a94cd0a6eca969a2466add4db9 to restore RIT 2024-12-07T07:13:36,861 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646861 2024-12-07T07:13:36,862 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,862 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,863 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,864 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,864 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,865 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,869 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,869 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676869 2024-12-07T07:13:36,869 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736869 2024-12-07T07:13:36,869 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-07T07:13:36,869 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,869 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,869 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,869 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,869 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,869 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,870 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,870 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,871 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,873 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,875 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,875 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,875 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,875 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,875 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,875 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,876 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,876 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN 2024-12-07T07:13:36,876 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-07T07:13:36,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,877 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,877 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,890 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,890 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,890 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab678da1dd813ac72686a84c4f22fdc0, ASSIGN to state=OFFLINE, location=null, table=test, region=ab678da1dd813ac72686a84c4f22fdc0 to restore RIT 2024-12-07T07:13:36,890 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646890 2024-12-07T07:13:36,890 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,890 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,891 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,892 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,892 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,898 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,898 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676898 2024-12-07T07:13:36,898 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736898 2024-12-07T07:13:36,898 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-07T07:13:36,898 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,898 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,898 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,898 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,898 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,898 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,898 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,899 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,899 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,901 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,902 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,903 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,903 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,903 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,903 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,903 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,903 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,904 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,904 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN 2024-12-07T07:13:36,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-07T07:13:36,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,904 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,905 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,909 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,909 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,914 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,914 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,914 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,914 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,914 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,914 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7, ASSIGN to state=OFFLINE, location=null, table=test, region=d07cc4fd0d69fc55b90b071c1b0400c7 to restore RIT 2024-12-07T07:13:36,914 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646914 2024-12-07T07:13:36,914 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,914 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,916 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,919 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,919 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,919 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,921 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,921 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676921 2024-12-07T07:13:36,921 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736921 2024-12-07T07:13:36,921 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-07T07:13:36,921 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,921 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,922 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,922 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,922 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,922 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,922 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,922 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,923 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN 2024-12-07T07:13:36,926 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T07:13:36,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,927 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,927 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,935 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,935 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,935 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,935 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,935 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ASSIGN to state=OFFLINE, location=null, table=test, region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3 to restore RIT 2024-12-07T07:13:36,935 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646935 2024-12-07T07:13:36,935 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,935 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,937 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,937 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,938 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,938 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,944 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,944 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676944 2024-12-07T07:13:36,944 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736944 2024-12-07T07:13:36,944 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-07T07:13:36,944 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,944 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,944 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,944 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,944 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,944 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,944 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,945 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN 2024-12-07T07:13:36,949 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T07:13:36,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,950 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,950 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,963 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,963 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,963 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,963 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=acd8e0c860fdbce58ce803c15220afb0, ASSIGN to state=OFFLINE, location=null, table=test, region=acd8e0c860fdbce58ce803c15220afb0 to restore RIT 2024-12-07T07:13:36,963 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646963 2024-12-07T07:13:36,963 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,964 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,965 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,967 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,967 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,967 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,971 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,972 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676971 2024-12-07T07:13:36,972 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736972 2024-12-07T07:13:36,972 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-07T07:13:36,972 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,972 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,972 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,972 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,972 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,972 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,972 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,972 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,973 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,975 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,978 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:36,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:36,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,978 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:36,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN 2024-12-07T07:13:36,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-07T07:13:36,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:36,979 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:36,979 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:36,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:36,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:36,987 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:36,987 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:36,987 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a, ASSIGN to state=OFFLINE, location=null, table=test, region=c1e55fd1b7105eca2dcbd1d5897f7f5a to restore RIT 2024-12-07T07:13:36,988 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555646988 2024-12-07T07:13:36,988 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:36,988 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,989 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:36,990 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:36,990 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:36,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,990 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:36,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,992 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,993 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,993 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,993 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,993 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,993 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,993 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,993 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,993 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,994 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:36,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:36,995 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:36,995 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555676995 2024-12-07T07:13:36,995 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555736995 2024-12-07T07:13:36,995 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-07T07:13:36,995 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:36,995 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:36,995 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:36,995 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:36,996 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:36,996 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:36,996 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:36,996 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:36,997 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:36,998 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:36,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:36,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:36,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:36,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:36,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:36,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:37,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,001 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN 2024-12-07T07:13:37,002 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-07T07:13:37,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:37,002 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:37,003 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:37,015 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,015 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:37,177 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2cfa28a210d48360878dfe1333fd8e26, ASSIGN to state=OFFLINE, location=null, table=test, region=2cfa28a210d48360878dfe1333fd8e26 to restore RIT 2024-12-07T07:13:37,177 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555647177 2024-12-07T07:13:37,178 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:37,178 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,180 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:37,181 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:37,181 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:37,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,182 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,186 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:37,186 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555677186 2024-12-07T07:13:37,186 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555737186 2024-12-07T07:13:37,186 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-07T07:13:37,186 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:37,186 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,186 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:37,186 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:37,186 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:37,186 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:37,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:37,187 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:37,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:37,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:37,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:37,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:37,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:37,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:37,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:37,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:37,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:37,189 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:37,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,191 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,191 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN 2024-12-07T07:13:37,191 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T07:13:37,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:37,192 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:37,192 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:37,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,241 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,241 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,241 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,241 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fe956aa50065b9ae6300ba6efb3d715d, ASSIGN to state=OFFLINE, location=null, table=test, region=fe956aa50065b9ae6300ba6efb3d715d to restore RIT 2024-12-07T07:13:37,242 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555647242 2024-12-07T07:13:37,242 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:37,242 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,243 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:37,243 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:37,244 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,244 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,247 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:37,247 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555677247 2024-12-07T07:13:37,248 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555737248 2024-12-07T07:13:37,248 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-07T07:13:37,248 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:37,248 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,248 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:37,248 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:37,248 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:37,248 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:37,248 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:37,248 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:37,249 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:37,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:37,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:37,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:37,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:37,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:37,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:37,250 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:37,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:37,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:37,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:37,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,252 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-07T07:13:37,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:37,253 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:37,253 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:37,269 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T07:13:37,278 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,278 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,281 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,281 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,281 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,284 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555647284 2024-12-07T07:13:37,284 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:37,284 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,286 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:37,287 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:37,287 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,287 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,293 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:37,293 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555677293 2024-12-07T07:13:37,293 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555737293 2024-12-07T07:13:37,293 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-07T07:13:37,293 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:37,293 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,293 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:37,294 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:37,294 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:37,294 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:37,294 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:37,294 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:37,295 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:37,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:37,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:37,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:37,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:37,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:37,297 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:37,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:37,298 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,299 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.1733555611238, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,299 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-07T07:13:37,300 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:37,300 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:37,300 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:37,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,314 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555647314 2024-12-07T07:13:37,314 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:37,314 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,316 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:37,318 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:37,318 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,318 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,321 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:37,321 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555677321 2024-12-07T07:13:37,321 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555737321 2024-12-07T07:13:37,321 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-07T07:13:37,321 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:37,321 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,321 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:37,321 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:37,321 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:37,322 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:37,322 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:37,322 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-12-07T07:13:37,322 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:37,323 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:37,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:37,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:37,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:37,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:37,324 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-07T07:13:37,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:37,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:37,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:37,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:37,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,325 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-07T07:13:37,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:37,326 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:37,326 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:37,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,334 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,334 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,334 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,335 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555647335 2024-12-07T07:13:37,335 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:37,335 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,336 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:37,337 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:37,337 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:37,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,337 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,341 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:37,341 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555677341 2024-12-07T07:13:37,341 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555737341 2024-12-07T07:13:37,341 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-07T07:13:37,341 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:37,341 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,341 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:37,341 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:37,341 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:37,341 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:37,341 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:37,342 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:37,342 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:37,343 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:37,343 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:37,343 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:37,343 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-07T07:13:37,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:37,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:37,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:37,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T07:13:37,345 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,345 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,345 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,345 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,345 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-07T07:13:37,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:37,345 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:37,345 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:37,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,355 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,355 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,355 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,356 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555647356 2024-12-07T07:13:37,356 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:37,356 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,357 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:37,358 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:37,358 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,358 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,361 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,362 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,362 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:37,362 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555677362 2024-12-07T07:13:37,362 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555737362 2024-12-07T07:13:37,362 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-07T07:13:37,362 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:37,362 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,362 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:37,362 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:37,363 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:37,363 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:37,363 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:37,363 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:37,364 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:37,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:37,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:37,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-07T07:13:37,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:37,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:37,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:37,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,366 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,366 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-07T07:13:37,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:37,367 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:37,367 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:37,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,376 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,376 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,376 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,377 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555647377 2024-12-07T07:13:37,377 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:37,377 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,378 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:37,379 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:37,379 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,379 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,382 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:37,382 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555677382 2024-12-07T07:13:37,382 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555737382 2024-12-07T07:13:37,382 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-07T07:13:37,382 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:37,382 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,383 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:37,383 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:37,383 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:37,383 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:37,383 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:37,383 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:37,383 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,384 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:37,384 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46437 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,386 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:37,386 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-07T07:13:37,386 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:37,386 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:37,387 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:37,387 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,387 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,387 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 5d4d7471f19e%2C41097%2C1733555609760.meta.1733555611725.meta, worker=5d4d7471f19e,34103,1733555609616 didn't hold the lock before restarting, skip acquiring lock. 2024-12-07T07:13:37,387 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T07:13:37,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:37,388 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:37,388 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:37,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,398 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555647398 2024-12-07T07:13:37,398 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:37,398 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,399 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:37,400 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-07T07:13:37,400 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:37,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,400 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,404 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:37,404 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555677404 2024-12-07T07:13:37,404 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555737404 2024-12-07T07:13:37,404 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-07T07:13:37,404 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-07T07:13:37,404 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,404 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:37,404 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:37,404 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-07T07:13:37,404 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-07T07:13:37,404 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-07T07:13:37,404 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T07:13:37,405 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T07:13:37,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-07T07:13:37,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T07:13:37,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-07T07:13:37,408 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true 2024-12-07T07:13:37,408 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-07T07:13:37,408 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-07T07:13:37,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=61 2024-12-07T07:13:37,409 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=5d4d7471f19e,34103,1733555609616, table=hbase:meta, region=1588230740 2024-12-07T07:13:37,409 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5d4d7471f19e,34103,1733555609616, state=OPEN 2024-12-07T07:13:37,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T07:13:37,418 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,418 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,418 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T07:13:37,419 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733555647419 2024-12-07T07:13:37,419 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T07:13:37,419 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-07T07:13:37,421 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-07T07:13:37,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure c1e55fd1b7105eca2dcbd1d5897f7f5a, server=5d4d7471f19e,34103,1733555609616:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 5d4d7471f19e,41097,1733555609760, splitWal=true, meta=true exec-time=2.7810 sec 2024-12-07T07:13:37,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff1807b6b64d6c4ca2c6c6d7a2cafbd3, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=58e2442015eb81a1979c8a3c79c8b4a4, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf320858c98aa00078ada211b555e73e, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=764f719c1b7c5872a4cc2b9d21f2dc24, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6aadffb1e0d6160805fffee563de94ca, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acd8e0c860fdbce58ce803c15220afb0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5101f15d587bc41cd49e68708e4a5ed2, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caa1b434450bd250c5e2792eafa50df0, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cfa28a210d48360878dfe1333fd8e26, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51e3bd49877ea2b244de43a10f4eaa37, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a24f56f7916756473277b23c11f5ce84, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fe956aa50065b9ae6300ba6efb3d715d, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c8344bda7e0c92708aba9a317b54627, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e7ad8954e0a4d51f932a71d965426b, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1e55fd1b7105eca2dcbd1d5897f7f5a, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c709bd9f88fc6dff6480583f422247fa, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=37e2dfda6af2ae02ec7c14d6781d4503, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffbaf4a5440811c93c49fb27af711a24, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab678da1dd813ac72686a84c4f22fdc0, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da5554a80a6c43e6ba1b1014d9f4023a, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5369f3b9703dae7de70d6556989c74a8, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9d2616a94cd0a6eca969a2466add4db9, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99de37e290b4841263ba62805b625d97, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d60e3c4175109e97854d2b36c43a459f, regionState=OPEN, lastHost=5d4d7471f19e,34103,1733555609616, regionLocation=5d4d7471f19e,34103,1733555609616, openSeqNum=2 2024-12-07T07:13:37,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d07cc4fd0d69fc55b90b071c1b0400c7, regionState=OPENING, lastHost=5d4d7471f19e,41097,1733555609760, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abe9b4a81ffb7c674120d136c6ab6231, regionState=OPEN, lastHost=5d4d7471f19e,37405,1733555609713, regionLocation=5d4d7471f19e,37405,1733555609713, openSeqNum=2 2024-12-07T07:13:37,425 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T07:13:37,425 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733555677425 2024-12-07T07:13:37,425 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733555737425 2024-12-07T07:13:37,425 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-07T07:13:37,442 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=346 (was 380), OpenFileDescriptor=614 (was 637), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=201 (was 219), ProcessCount=11 (was 11), AvailableMemoryMB=6994 (was 7049) 2024-12-07T07:13:37,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T07:13:37,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T07:13:37,442 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T07:13:37,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T07:13:37,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T07:13:37,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T07:13:37,443 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T07:13:37,443 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1780335685, stopped=false 2024-12-07T07:13:37,443 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5d4d7471f19e,46437,1733555608905 2024-12-07T07:13:37,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T07:13:37,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T07:13:37,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T07:13:37,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:37,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:37,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:37,450 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,450 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T07:13:37,450 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T07:13:37,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T07:13:37,450 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5d4d7471f19e,34103,1733555609616' ***** 2024-12-07T07:13:37,450 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T07:13:37,450 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T07:13:37,450 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5d4d7471f19e,37405,1733555609713' ***** 2024-12-07T07:13:37,450 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T07:13:37,450 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T07:13:37,450 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T07:13:37,450 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T07:13:37,451 INFO [RS:0;5d4d7471f19e:34103 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T07:13:37,451 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T07:13:37,451 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T07:13:37,451 INFO [RS:0;5d4d7471f19e:34103 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T07:13:37,451 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:37,451 INFO [RS:1;5d4d7471f19e:37405 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T07:13:37,451 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T07:13:37,451 INFO [RS:1;5d4d7471f19e:37405 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T07:13:37,451 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for 99de37e290b4841263ba62805b625d97 2024-12-07T07:13:37,451 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for c1e55fd1b7105eca2dcbd1d5897f7f5a 2024-12-07T07:13:37,451 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for ff1807b6b64d6c4ca2c6c6d7a2cafbd3 2024-12-07T07:13:37,451 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for 4c8344bda7e0c92708aba9a317b54627 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for d60e3c4175109e97854d2b36c43a459f 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for caa1b434450bd250c5e2792eafa50df0 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for 2cfa28a210d48360878dfe1333fd8e26 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for 5101f15d587bc41cd49e68708e4a5ed2 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for da5554a80a6c43e6ba1b1014d9f4023a 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for 45e7ad8954e0a4d51f932a71d965426b 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for 58e2442015eb81a1979c8a3c79c8b4a4 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for cf320858c98aa00078ada211b555e73e 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for d07cc4fd0d69fc55b90b071c1b0400c7 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for 5369f3b9703dae7de70d6556989c74a8 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for ab678da1dd813ac72686a84c4f22fdc0 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for 51e3bd49877ea2b244de43a10f4eaa37 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for 764f719c1b7c5872a4cc2b9d21f2dc24 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for 6aadffb1e0d6160805fffee563de94ca 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for 9d2616a94cd0a6eca969a2466add4db9 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for a24f56f7916756473277b23c11f5ce84 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for 37e2dfda6af2ae02ec7c14d6781d4503 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for c709bd9f88fc6dff6480583f422247fa 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for abe9b4a81ffb7c674120d136c6ab6231 2024-12-07T07:13:37,452 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 99de37e290b4841263ba62805b625d97, disabling compactions & flushes 2024-12-07T07:13:37,452 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ffbaf4a5440811c93c49fb27af711a24, disabling compactions & flushes 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(959): stopping server 5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(3091): Received CLOSE for acd8e0c860fdbce58ce803c15220afb0 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(959): stopping server 5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:37,452 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:37,452 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T07:13:37,452 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:37,452 INFO [RS:0;5d4d7471f19e:34103 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5d4d7471f19e:34103. 2024-12-07T07:13:37,452 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:37,452 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1733555612106.99de37e290b4841263ba62805b625d97. after waiting 0 ms 2024-12-07T07:13:37,452 INFO [RS:1;5d4d7471f19e:37405 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;5d4d7471f19e:37405. 2024-12-07T07:13:37,452 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:37,452 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. after waiting 0 ms 2024-12-07T07:13:37,452 DEBUG [RS:0;5d4d7471f19e:34103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T07:13:37,452 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:37,452 DEBUG [RS:1;5d4d7471f19e:37405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T07:13:37,452 DEBUG [RS:0;5d4d7471f19e:34103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T07:13:37,452 DEBUG [RS:1;5d4d7471f19e:37405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T07:13:37,453 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-12-07T07:13:37,453 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T07:13:37,453 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T07:13:37,453 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T07:13:37,453 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T07:13:37,453 DEBUG [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(1351): Waiting on 4c8344bda7e0c92708aba9a317b54627, 58e2442015eb81a1979c8a3c79c8b4a4, 764f719c1b7c5872a4cc2b9d21f2dc24, 99de37e290b4841263ba62805b625d97, a24f56f7916756473277b23c11f5ce84, ab678da1dd813ac72686a84c4f22fdc0, abe9b4a81ffb7c674120d136c6ab6231, acd8e0c860fdbce58ce803c15220afb0, c709bd9f88fc6dff6480583f422247fa, caa1b434450bd250c5e2792eafa50df0, d07cc4fd0d69fc55b90b071c1b0400c7, da5554a80a6c43e6ba1b1014d9f4023a, fe956aa50065b9ae6300ba6efb3d715d 2024-12-07T07:13:37,453 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-12-07T07:13:37,453 DEBUG [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2cfa28a210d48360878dfe1333fd8e26, 37e2dfda6af2ae02ec7c14d6781d4503, 45e7ad8954e0a4d51f932a71d965426b, 5101f15d587bc41cd49e68708e4a5ed2, 51e3bd49877ea2b244de43a10f4eaa37, 5369f3b9703dae7de70d6556989c74a8, 6aadffb1e0d6160805fffee563de94ca, 9d2616a94cd0a6eca969a2466add4db9, c1e55fd1b7105eca2dcbd1d5897f7f5a, cf320858c98aa00078ada211b555e73e, d60e3c4175109e97854d2b36c43a459f, ff1807b6b64d6c4ca2c6c6d7a2cafbd3, ffbaf4a5440811c93c49fb27af711a24 2024-12-07T07:13:37,453 DEBUG [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T07:13:37,453 INFO [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T07:13:37,453 DEBUG [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T07:13:37,453 DEBUG [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T07:13:37,453 DEBUG [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T07:13:37,454 INFO [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-12-07T07:13:37,457 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/99de37e290b4841263ba62805b625d97/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,458 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ffbaf4a5440811c93c49fb27af711a24/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,458 INFO [regionserver/5d4d7471f19e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T07:13:37,459 INFO [regionserver/5d4d7471f19e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T07:13:37,460 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:37,460 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:37,460 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ffbaf4a5440811c93c49fb27af711a24: Waiting for close lock at 1733555617452Running coprocessor pre-close hooks at 1733555617452Disabling compacts and flushes for region at 1733555617452Disabling writes for close at 1733555617452Writing region close event to WAL at 1733555617453 (+1 ms)Running coprocessor post-close hooks at 1733555617458 (+5 ms)Closed at 1733555617460 (+2 ms) 2024-12-07T07:13:37,460 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 99de37e290b4841263ba62805b625d97: Waiting for close lock at 1733555617452Running coprocessor pre-close hooks at 1733555617452Disabling compacts and flushes for region at 1733555617452Disabling writes for close at 1733555617452Writing region close event to WAL at 1733555617453 (+1 ms)Running coprocessor post-close hooks at 1733555617458 (+5 ms)Closed at 1733555617460 (+2 ms) 2024-12-07T07:13:37,460 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733555612106.ffbaf4a5440811c93c49fb27af711a24. 2024-12-07T07:13:37,460 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1733555612106.99de37e290b4841263ba62805b625d97. 2024-12-07T07:13:37,460 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c1e55fd1b7105eca2dcbd1d5897f7f5a, disabling compactions & flushes 2024-12-07T07:13:37,460 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4c8344bda7e0c92708aba9a317b54627, disabling compactions & flushes 2024-12-07T07:13:37,460 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:37,460 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:37,460 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:37,461 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. after waiting 0 ms 2024-12-07T07:13:37,461 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:37,461 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:37,461 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. after waiting 0 ms 2024-12-07T07:13:37,461 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:37,465 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/c1e55fd1b7105eca2dcbd1d5897f7f5a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,466 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/4c8344bda7e0c92708aba9a317b54627/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,466 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:37,466 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:37,466 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c1e55fd1b7105eca2dcbd1d5897f7f5a: Waiting for close lock at 1733555617460Running coprocessor pre-close hooks at 1733555617460Disabling compacts and flushes for region at 1733555617460Disabling writes for close at 1733555617461 (+1 ms)Writing region close event to WAL at 1733555617461Running coprocessor post-close hooks at 1733555617466 (+5 ms)Closed at 1733555617466 2024-12-07T07:13:37,466 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4c8344bda7e0c92708aba9a317b54627: Waiting for close lock at 1733555617460Running coprocessor pre-close hooks at 1733555617460Disabling compacts and flushes for region at 1733555617460Disabling writes for close at 1733555617461 (+1 ms)Writing region close event to WAL at 1733555617461Running coprocessor post-close hooks at 1733555617466 (+5 ms)Closed at 1733555617466 2024-12-07T07:13:37,467 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733555612106.c1e55fd1b7105eca2dcbd1d5897f7f5a. 2024-12-07T07:13:37,467 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1733555612106.4c8344bda7e0c92708aba9a317b54627. 2024-12-07T07:13:37,467 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff1807b6b64d6c4ca2c6c6d7a2cafbd3, disabling compactions & flushes 2024-12-07T07:13:37,467 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:37,467 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing caa1b434450bd250c5e2792eafa50df0, disabling compactions & flushes 2024-12-07T07:13:37,467 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:37,467 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:37,467 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. after waiting 0 ms 2024-12-07T07:13:37,467 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:37,467 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:37,467 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. after waiting 0 ms 2024-12-07T07:13:37,467 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:37,468 DEBUG [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/info/6a0f7e1a5def4e0e93aa09750781a991 is 138, key is test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0./info:regioninfo/1733555616129/Put/seqid=0 2024-12-07T07:13:37,472 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ff1807b6b64d6c4ca2c6c6d7a2cafbd3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,472 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:37,473 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff1807b6b64d6c4ca2c6c6d7a2cafbd3: Waiting for close lock at 1733555617467Running coprocessor pre-close hooks at 1733555617467Disabling compacts and flushes for region at 1733555617467Disabling writes for close at 1733555617467Writing region close event to WAL at 1733555617467Running coprocessor post-close hooks at 1733555617472 (+5 ms)Closed at 1733555617472 2024-12-07T07:13:37,473 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1733555612106.ff1807b6b64d6c4ca2c6c6d7a2cafbd3. 2024-12-07T07:13:37,473 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d60e3c4175109e97854d2b36c43a459f, disabling compactions & flushes 2024-12-07T07:13:37,473 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:37,473 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:37,473 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. after waiting 0 ms 2024-12-07T07:13:37,473 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:37,474 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/caa1b434450bd250c5e2792eafa50df0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,475 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:37,475 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for caa1b434450bd250c5e2792eafa50df0: Waiting for close lock at 1733555617467Running coprocessor pre-close hooks at 1733555617467Disabling compacts and flushes for region at 1733555617467Disabling writes for close at 1733555617467Writing region close event to WAL at 1733555617469 (+2 ms)Running coprocessor post-close hooks at 1733555617475 (+6 ms)Closed at 1733555617475 2024-12-07T07:13:37,475 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733555612106.caa1b434450bd250c5e2792eafa50df0. 2024-12-07T07:13:37,475 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing da5554a80a6c43e6ba1b1014d9f4023a, disabling compactions & flushes 2024-12-07T07:13:37,475 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:37,476 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:37,476 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. after waiting 0 ms 2024-12-07T07:13:37,476 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:37,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741869_1045 (size=8177) 2024-12-07T07:13:37,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741869_1045 (size=8177) 2024-12-07T07:13:37,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741869_1045 (size=8177) 2024-12-07T07:13:37,479 INFO [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/info/6a0f7e1a5def4e0e93aa09750781a991 2024-12-07T07:13:37,479 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/d60e3c4175109e97854d2b36c43a459f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,480 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:37,481 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d60e3c4175109e97854d2b36c43a459f: Waiting for close lock at 1733555617473Running coprocessor pre-close hooks at 1733555617473Disabling compacts and flushes for region at 1733555617473Disabling writes for close at 1733555617473Writing region close event to WAL at 1733555617474 (+1 ms)Running coprocessor post-close hooks at 1733555617480 (+6 ms)Closed at 1733555617480 2024-12-07T07:13:37,481 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733555612106.d60e3c4175109e97854d2b36c43a459f. 2024-12-07T07:13:37,481 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2cfa28a210d48360878dfe1333fd8e26, disabling compactions & flushes 2024-12-07T07:13:37,481 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:37,481 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:37,481 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. after waiting 0 ms 2024-12-07T07:13:37,481 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:37,481 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/da5554a80a6c43e6ba1b1014d9f4023a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,482 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:37,483 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for da5554a80a6c43e6ba1b1014d9f4023a: Waiting for close lock at 1733555617475Running coprocessor pre-close hooks at 1733555617475Disabling compacts and flushes for region at 1733555617475Disabling writes for close at 1733555617476 (+1 ms)Writing region close event to WAL at 1733555617476Running coprocessor post-close hooks at 1733555617482 (+6 ms)Closed at 1733555617482 2024-12-07T07:13:37,483 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733555612106.da5554a80a6c43e6ba1b1014d9f4023a. 2024-12-07T07:13:37,483 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 58e2442015eb81a1979c8a3c79c8b4a4, disabling compactions & flushes 2024-12-07T07:13:37,483 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:37,483 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:37,483 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. after waiting 0 ms 2024-12-07T07:13:37,483 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:37,487 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/2cfa28a210d48360878dfe1333fd8e26/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,487 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:37,487 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2cfa28a210d48360878dfe1333fd8e26: Waiting for close lock at 1733555617481Running coprocessor pre-close hooks at 1733555617481Disabling compacts and flushes for region at 1733555617481Disabling writes for close at 1733555617481Writing region close event to WAL at 1733555617482 (+1 ms)Running coprocessor post-close hooks at 1733555617487 (+5 ms)Closed at 1733555617487 2024-12-07T07:13:37,488 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1733555612106.2cfa28a210d48360878dfe1333fd8e26. 2024-12-07T07:13:37,488 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/58e2442015eb81a1979c8a3c79c8b4a4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,488 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5101f15d587bc41cd49e68708e4a5ed2, disabling compactions & flushes 2024-12-07T07:13:37,488 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:37,488 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:37,488 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. after waiting 0 ms 2024-12-07T07:13:37,488 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:37,488 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:37,489 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 58e2442015eb81a1979c8a3c79c8b4a4: Waiting for close lock at 1733555617483Running coprocessor pre-close hooks at 1733555617483Disabling compacts and flushes for region at 1733555617483Disabling writes for close at 1733555617483Writing region close event to WAL at 1733555617484 (+1 ms)Running coprocessor post-close hooks at 1733555617488 (+4 ms)Closed at 1733555617488 2024-12-07T07:13:37,489 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1733555612106.58e2442015eb81a1979c8a3c79c8b4a4. 2024-12-07T07:13:37,489 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d07cc4fd0d69fc55b90b071c1b0400c7, disabling compactions & flushes 2024-12-07T07:13:37,489 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:37,489 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:37,489 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. after waiting 0 ms 2024-12-07T07:13:37,489 DEBUG [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/.tmp/info/6a0f7e1a5def4e0e93aa09750781a991 as hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/info/6a0f7e1a5def4e0e93aa09750781a991 2024-12-07T07:13:37,489 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:37,494 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/d07cc4fd0d69fc55b90b071c1b0400c7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,494 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/5101f15d587bc41cd49e68708e4a5ed2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,495 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:37,495 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d07cc4fd0d69fc55b90b071c1b0400c7: Waiting for close lock at 1733555617489Running coprocessor pre-close hooks at 1733555617489Disabling compacts and flushes for region at 1733555617489Disabling writes for close at 1733555617489Writing region close event to WAL at 1733555617490 (+1 ms)Running coprocessor post-close hooks at 1733555617495 (+5 ms)Closed at 1733555617495 2024-12-07T07:13:37,495 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1733555612106.d07cc4fd0d69fc55b90b071c1b0400c7. 2024-12-07T07:13:37,495 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:37,495 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5101f15d587bc41cd49e68708e4a5ed2: Waiting for close lock at 1733555617488Running coprocessor pre-close hooks at 1733555617488Disabling compacts and flushes for region at 1733555617488Disabling writes for close at 1733555617488Writing region close event to WAL at 1733555617488Running coprocessor post-close hooks at 1733555617495 (+7 ms)Closed at 1733555617495 2024-12-07T07:13:37,495 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab678da1dd813ac72686a84c4f22fdc0, disabling compactions & flushes 2024-12-07T07:13:37,495 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:37,495 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:37,495 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733555612106.5101f15d587bc41cd49e68708e4a5ed2. 2024-12-07T07:13:37,495 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. after waiting 0 ms 2024-12-07T07:13:37,495 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:37,495 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 45e7ad8954e0a4d51f932a71d965426b, disabling compactions & flushes 2024-12-07T07:13:37,496 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:37,496 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:37,496 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. after waiting 0 ms 2024-12-07T07:13:37,496 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:37,500 INFO [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/info/6a0f7e1a5def4e0e93aa09750781a991, entries=24, sequenceid=71, filesize=8.0 K 2024-12-07T07:13:37,501 INFO [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 48ms, sequenceid=71, compaction requested=false 2024-12-07T07:13:37,502 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/ab678da1dd813ac72686a84c4f22fdc0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,502 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/45e7ad8954e0a4d51f932a71d965426b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,503 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 45e7ad8954e0a4d51f932a71d965426b: Waiting for close lock at 1733555617495Running coprocessor pre-close hooks at 1733555617495Disabling compacts and flushes for region at 1733555617495Disabling writes for close at 1733555617496 (+1 ms)Writing region close event to WAL at 1733555617497 (+1 ms)Running coprocessor post-close hooks at 1733555617502 (+5 ms)Closed at 1733555617503 (+1 ms) 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733555612106.45e7ad8954e0a4d51f932a71d965426b. 2024-12-07T07:13:37,503 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab678da1dd813ac72686a84c4f22fdc0: Waiting for close lock at 1733555617495Running coprocessor pre-close hooks at 1733555617495Disabling compacts and flushes for region at 1733555617495Disabling writes for close at 1733555617495Writing region close event to WAL at 1733555617496 (+1 ms)Running coprocessor post-close hooks at 1733555617503 (+7 ms)Closed at 1733555617503 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cf320858c98aa00078ada211b555e73e, disabling compactions & flushes 2024-12-07T07:13:37,503 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. after waiting 0 ms 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733555612106.ab678da1dd813ac72686a84c4f22fdc0. 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 764f719c1b7c5872a4cc2b9d21f2dc24, disabling compactions & flushes 2024-12-07T07:13:37,503 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. after waiting 0 ms 2024-12-07T07:13:37,503 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:37,509 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/cf320858c98aa00078ada211b555e73e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,510 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:37,510 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cf320858c98aa00078ada211b555e73e: Waiting for close lock at 1733555617503Running coprocessor pre-close hooks at 1733555617503Disabling compacts and flushes for region at 1733555617503Disabling writes for close at 1733555617503Writing region close event to WAL at 1733555617504 (+1 ms)Running coprocessor post-close hooks at 1733555617510 (+6 ms)Closed at 1733555617510 2024-12-07T07:13:37,510 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1733555612106.cf320858c98aa00078ada211b555e73e. 2024-12-07T07:13:37,511 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5369f3b9703dae7de70d6556989c74a8, disabling compactions & flushes 2024-12-07T07:13:37,511 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:37,511 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:37,511 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. after waiting 0 ms 2024-12-07T07:13:37,511 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:37,513 DEBUG [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-12-07T07:13:37,514 DEBUG [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T07:13:37,514 INFO [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T07:13:37,514 DEBUG [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733555617453Running coprocessor pre-close hooks at 1733555617453Disabling compacts and flushes for region at 1733555617453Disabling writes for close at 1733555617453Obtaining lock to block concurrent updates at 1733555617454 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733555617454Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1733555617461 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733555617463 (+2 ms)Flushing 1588230740/info: creating writer at 1733555617463Flushing 1588230740/info: appending metadata at 1733555617468 (+5 ms)Flushing 1588230740/info: closing flushed file at 1733555617468Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e8516a0: reopening flushed file at 1733555617488 (+20 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 48ms, sequenceid=71, compaction requested=false at 1733555617501 (+13 ms)Writing region close event to WAL at 1733555617508 (+7 ms)Running coprocessor post-close hooks at 1733555617513 (+5 ms)Closed at 1733555617514 (+1 ms) 2024-12-07T07:13:37,514 DEBUG [RS_CLOSE_META-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T07:13:37,515 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/764f719c1b7c5872a4cc2b9d21f2dc24/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,516 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:37,516 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/5369f3b9703dae7de70d6556989c74a8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,516 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 764f719c1b7c5872a4cc2b9d21f2dc24: Waiting for close lock at 1733555617503Running coprocessor pre-close hooks at 1733555617503Disabling compacts and flushes for region at 1733555617503Disabling writes for close at 1733555617503Writing region close event to WAL at 1733555617508 (+5 ms)Running coprocessor post-close hooks at 1733555617515 (+7 ms)Closed at 1733555617515 2024-12-07T07:13:37,516 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733555612106.764f719c1b7c5872a4cc2b9d21f2dc24. 2024-12-07T07:13:37,516 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fe956aa50065b9ae6300ba6efb3d715d, disabling compactions & flushes 2024-12-07T07:13:37,516 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:37,516 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:37,516 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. after waiting 0 ms 2024-12-07T07:13:37,516 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:37,516 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:37,516 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5369f3b9703dae7de70d6556989c74a8: Waiting for close lock at 1733555617510Running coprocessor pre-close hooks at 1733555617510Disabling compacts and flushes for region at 1733555617510Disabling writes for close at 1733555617511 (+1 ms)Writing region close event to WAL at 1733555617511Running coprocessor post-close hooks at 1733555617516 (+5 ms)Closed at 1733555617516 2024-12-07T07:13:37,516 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733555612106.5369f3b9703dae7de70d6556989c74a8. 2024-12-07T07:13:37,517 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 51e3bd49877ea2b244de43a10f4eaa37, disabling compactions & flushes 2024-12-07T07:13:37,517 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:37,517 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:37,517 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. after waiting 0 ms 2024-12-07T07:13:37,517 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:37,521 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/51e3bd49877ea2b244de43a10f4eaa37/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,522 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/fe956aa50065b9ae6300ba6efb3d715d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,522 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 51e3bd49877ea2b244de43a10f4eaa37: Waiting for close lock at 1733555617517Running coprocessor pre-close hooks at 1733555617517Disabling compacts and flushes for region at 1733555617517Disabling writes for close at 1733555617517Writing region close event to WAL at 1733555617517Running coprocessor post-close hooks at 1733555617522 (+5 ms)Closed at 1733555617522 2024-12-07T07:13:37,523 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1733555612106.51e3bd49877ea2b244de43a10f4eaa37. 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fe956aa50065b9ae6300ba6efb3d715d: Waiting for close lock at 1733555617516Running coprocessor pre-close hooks at 1733555617516Disabling compacts and flushes for region at 1733555617516Disabling writes for close at 1733555617516Writing region close event to WAL at 1733555617517 (+1 ms)Running coprocessor post-close hooks at 1733555617523 (+6 ms)Closed at 1733555617523 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1733555612106.fe956aa50065b9ae6300ba6efb3d715d. 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6aadffb1e0d6160805fffee563de94ca, disabling compactions & flushes 2024-12-07T07:13:37,523 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a24f56f7916756473277b23c11f5ce84, disabling compactions & flushes 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. after waiting 0 ms 2024-12-07T07:13:37,523 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. after waiting 0 ms 2024-12-07T07:13:37,523 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:37,528 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/6aadffb1e0d6160805fffee563de94ca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,528 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/a24f56f7916756473277b23c11f5ce84/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,528 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:37,529 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6aadffb1e0d6160805fffee563de94ca: Waiting for close lock at 1733555617523Running coprocessor pre-close hooks at 1733555617523Disabling compacts and flushes for region at 1733555617523Disabling writes for close at 1733555617523Writing region close event to WAL at 1733555617523Running coprocessor post-close hooks at 1733555617528 (+5 ms)Closed at 1733555617528 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a24f56f7916756473277b23c11f5ce84: Waiting for close lock at 1733555617523Running coprocessor pre-close hooks at 1733555617523Disabling compacts and flushes for region at 1733555617523Disabling writes for close at 1733555617523Writing region close event to WAL at 1733555617524 (+1 ms)Running coprocessor post-close hooks at 1733555617528 (+4 ms)Closed at 1733555617528 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733555612106.6aadffb1e0d6160805fffee563de94ca. 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733555612106.a24f56f7916756473277b23c11f5ce84. 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9d2616a94cd0a6eca969a2466add4db9, disabling compactions & flushes 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c709bd9f88fc6dff6480583f422247fa, disabling compactions & flushes 2024-12-07T07:13:37,529 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:37,529 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. after waiting 0 ms 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. after waiting 0 ms 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:37,529 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:37,534 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/c709bd9f88fc6dff6480583f422247fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,534 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/9d2616a94cd0a6eca969a2466add4db9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,534 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c709bd9f88fc6dff6480583f422247fa: Waiting for close lock at 1733555617529Running coprocessor pre-close hooks at 1733555617529Disabling compacts and flushes for region at 1733555617529Disabling writes for close at 1733555617529Writing region close event to WAL at 1733555617529Running coprocessor post-close hooks at 1733555617534 (+5 ms)Closed at 1733555617534 2024-12-07T07:13:37,535 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1733555612106.c709bd9f88fc6dff6480583f422247fa. 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9d2616a94cd0a6eca969a2466add4db9: Waiting for close lock at 1733555617529Running coprocessor pre-close hooks at 1733555617529Disabling compacts and flushes for region at 1733555617529Disabling writes for close at 1733555617529Writing region close event to WAL at 1733555617529Running coprocessor post-close hooks at 1733555617535 (+6 ms)Closed at 1733555617535 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1733555612106.9d2616a94cd0a6eca969a2466add4db9. 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing abe9b4a81ffb7c674120d136c6ab6231, disabling compactions & flushes 2024-12-07T07:13:37,535 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 37e2dfda6af2ae02ec7c14d6781d4503, disabling compactions & flushes 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. after waiting 0 ms 2024-12-07T07:13:37,535 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. after waiting 0 ms 2024-12-07T07:13:37,535 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:37,540 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/abe9b4a81ffb7c674120d136c6ab6231/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,540 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/37e2dfda6af2ae02ec7c14d6781d4503/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,540 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:37,540 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for abe9b4a81ffb7c674120d136c6ab6231: Waiting for close lock at 1733555617535Running coprocessor pre-close hooks at 1733555617535Disabling compacts and flushes for region at 1733555617535Disabling writes for close at 1733555617535Writing region close event to WAL at 1733555617535Running coprocessor post-close hooks at 1733555617540 (+5 ms)Closed at 1733555617540 2024-12-07T07:13:37,541 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1733555612106.abe9b4a81ffb7c674120d136c6ab6231. 2024-12-07T07:13:37,541 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing acd8e0c860fdbce58ce803c15220afb0, disabling compactions & flushes 2024-12-07T07:13:37,541 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:37,541 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:37,541 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:37,541 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. after waiting 0 ms 2024-12-07T07:13:37,541 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:37,541 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 37e2dfda6af2ae02ec7c14d6781d4503: Waiting for close lock at 1733555617535Running coprocessor pre-close hooks at 1733555617535Disabling compacts and flushes for region at 1733555617535Disabling writes for close at 1733555617535Writing region close event to WAL at 1733555617536 (+1 ms)Running coprocessor post-close hooks at 1733555617541 (+5 ms)Closed at 1733555617541 2024-12-07T07:13:37,541 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1733555612106.37e2dfda6af2ae02ec7c14d6781d4503. 2024-12-07T07:13:37,545 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/data/default/test/acd8e0c860fdbce58ce803c15220afb0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T07:13:37,546 INFO [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:37,546 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for acd8e0c860fdbce58ce803c15220afb0: Waiting for close lock at 1733555617541Running coprocessor pre-close hooks at 1733555617541Disabling compacts and flushes for region at 1733555617541Disabling writes for close at 1733555617541Writing region close event to WAL at 1733555617541Running coprocessor post-close hooks at 1733555617546 (+5 ms)Closed at 1733555617546 2024-12-07T07:13:37,546 DEBUG [RS_CLOSE_REGION-regionserver/5d4d7471f19e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1733555612106.acd8e0c860fdbce58ce803c15220afb0. 2024-12-07T07:13:37,653 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(976): stopping server 5d4d7471f19e,37405,1733555609713; all regions closed. 2024-12-07T07:13:37,653 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(976): stopping server 5d4d7471f19e,34103,1733555609616; all regions closed. 2024-12-07T07:13:37,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741865_1041 (size=4192) 2024-12-07T07:13:37,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741833_1009 (size=7074) 2024-12-07T07:13:37,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741865_1041 (size=4192) 2024-12-07T07:13:37,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741833_1009 (size=7074) 2024-12-07T07:13:37,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741865_1041 (size=4192) 2024-12-07T07:13:37,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741833_1009 (size=7074) 2024-12-07T07:13:37,662 DEBUG [RS:0;5d4d7471f19e:34103 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/oldWALs 2024-12-07T07:13:37,663 INFO [RS:0;5d4d7471f19e:34103 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5d4d7471f19e%2C34103%2C1733555609616.meta:.meta(num 1733555615370) 2024-12-07T07:13:37,663 DEBUG [RS:1;5d4d7471f19e:37405 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/oldWALs 2024-12-07T07:13:37,663 INFO [RS:1;5d4d7471f19e:37405 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5d4d7471f19e%2C37405%2C1733555609713:(num 1733555611205) 2024-12-07T07:13:37,663 DEBUG [RS:1;5d4d7471f19e:37405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T07:13:37,663 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T07:13:37,663 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T07:13:37,664 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.ChoreService(370): Chore service for: regionserver/5d4d7471f19e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T07:13:37,664 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T07:13:37,664 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T07:13:37,664 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T07:13:37,664 INFO [regionserver/5d4d7471f19e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T07:13:37,664 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T07:13:37,664 INFO [RS:1;5d4d7471f19e:37405 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37405 2024-12-07T07:13:37,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741834_1010 (size=7064) 2024-12-07T07:13:37,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741834_1010 (size=7064) 2024-12-07T07:13:37,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741834_1010 (size=7064) 2024-12-07T07:13:37,668 DEBUG [RS:0;5d4d7471f19e:34103 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/oldWALs 2024-12-07T07:13:37,669 INFO [RS:0;5d4d7471f19e:34103 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5d4d7471f19e%2C34103%2C1733555609616:(num 1733555611205) 2024-12-07T07:13:37,669 DEBUG [RS:0;5d4d7471f19e:34103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T07:13:37,669 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T07:13:37,669 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T07:13:37,669 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.ChoreService(370): Chore service for: regionserver/5d4d7471f19e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T07:13:37,669 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T07:13:37,669 INFO [regionserver/5d4d7471f19e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T07:13:37,669 INFO [RS:0;5d4d7471f19e:34103 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34103 2024-12-07T07:13:37,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T07:13:37,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5d4d7471f19e,37405,1733555609713 2024-12-07T07:13:37,743 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T07:13:37,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5d4d7471f19e,34103,1733555609616 2024-12-07T07:13:37,772 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T07:13:37,783 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5d4d7471f19e,37405,1733555609713] 2024-12-07T07:13:37,804 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5d4d7471f19e,37405,1733555609713 already deleted, retry=false 2024-12-07T07:13:37,804 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5d4d7471f19e,37405,1733555609713 expired; onlineServers=1 2024-12-07T07:13:37,804 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5d4d7471f19e,34103,1733555609616] 2024-12-07T07:13:37,815 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5d4d7471f19e,34103,1733555609616 already deleted, retry=false 2024-12-07T07:13:37,815 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5d4d7471f19e,34103,1733555609616 expired; onlineServers=0 2024-12-07T07:13:37,815 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5d4d7471f19e,46437,1733555608905' ***** 2024-12-07T07:13:37,815 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T07:13:37,816 INFO [M:0;5d4d7471f19e:46437 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T07:13:37,816 INFO [M:0;5d4d7471f19e:46437 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T07:13:37,816 DEBUG [M:0;5d4d7471f19e:46437 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T07:13:37,816 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T07:13:37,817 DEBUG [M:0;5d4d7471f19e:46437 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T07:13:37,817 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster-HFileCleaner.large.0-1733555610898 {}] cleaner.HFileCleaner(306): Exit Thread[master/5d4d7471f19e:0:becomeActiveMaster-HFileCleaner.large.0-1733555610898,5,FailOnTimeoutGroup] 2024-12-07T07:13:37,817 DEBUG [master/5d4d7471f19e:0:becomeActiveMaster-HFileCleaner.small.0-1733555610900 {}] cleaner.HFileCleaner(306): Exit Thread[master/5d4d7471f19e:0:becomeActiveMaster-HFileCleaner.small.0-1733555610900,5,FailOnTimeoutGroup] 2024-12-07T07:13:37,817 INFO [M:0;5d4d7471f19e:46437 {}] hbase.ChoreService(370): Chore service for: master/5d4d7471f19e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T07:13:37,817 INFO [M:0;5d4d7471f19e:46437 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T07:13:37,818 DEBUG [M:0;5d4d7471f19e:46437 {}] master.HMaster(1795): Stopping service threads 2024-12-07T07:13:37,818 INFO [M:0;5d4d7471f19e:46437 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T07:13:37,819 INFO [M:0;5d4d7471f19e:46437 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T07:13:37,819 INFO [M:0;5d4d7471f19e:46437 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T07:13:37,819 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T07:13:37,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T07:13:37,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T07:13:37,829 DEBUG [M:0;5d4d7471f19e:46437 {}] zookeeper.ZKUtil(347): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T07:13:37,829 WARN [M:0;5d4d7471f19e:46437 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T07:13:37,830 INFO [M:0;5d4d7471f19e:46437 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/.lastflushedseqids 2024-12-07T07:13:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741870_1046 (size=1) 2024-12-07T07:13:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741870_1046 (size=1) 2024-12-07T07:13:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741870_1046 (size=1) 2024-12-07T07:13:37,841 INFO [M:0;5d4d7471f19e:46437 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T07:13:37,841 INFO [M:0;5d4d7471f19e:46437 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T07:13:37,841 DEBUG [M:0;5d4d7471f19e:46437 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T07:13:37,841 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T07:13:37,841 DEBUG [M:0;5d4d7471f19e:46437 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T07:13:37,841 DEBUG [M:0;5d4d7471f19e:46437 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T07:13:37,841 DEBUG [M:0;5d4d7471f19e:46437 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T07:13:37,841 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=250.07 KB heapSize=303.92 KB 2024-12-07T07:13:37,855 DEBUG [M:0;5d4d7471f19e:46437 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/858e2b2150934a65b06f79ca0a1cfe6d is 82, key is hbase:meta,,1/info:regioninfo/1733555615693/Put/seqid=0 2024-12-07T07:13:37,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741871_1047 (size=6071) 2024-12-07T07:13:37,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741871_1047 (size=6071) 2024-12-07T07:13:37,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741871_1047 (size=6071) 2024-12-07T07:13:37,862 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/858e2b2150934a65b06f79ca0a1cfe6d 2024-12-07T07:13:37,883 INFO [RS:1;5d4d7471f19e:37405 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T07:13:37,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T07:13:37,883 INFO [RS:1;5d4d7471f19e:37405 {}] regionserver.HRegionServer(1031): Exiting; stopping=5d4d7471f19e,37405,1733555609713; zookeeper connection closed. 2024-12-07T07:13:37,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37405-0x101aea355730002, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T07:13:37,884 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@425c9593 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@425c9593 2024-12-07T07:13:37,885 DEBUG [M:0;5d4d7471f19e:46437 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/554dfdc37dac4bb5a66df6ba5592efc9 is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1733555617421/Put/seqid=0 2024-12-07T07:13:37,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741872_1048 (size=12799) 2024-12-07T07:13:37,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741872_1048 (size=12799) 2024-12-07T07:13:37,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741872_1048 (size=12799) 2024-12-07T07:13:37,893 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.80 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/554dfdc37dac4bb5a66df6ba5592efc9 2024-12-07T07:13:37,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T07:13:37,894 INFO [RS:0;5d4d7471f19e:34103 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T07:13:37,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34103-0x101aea355730001, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T07:13:37,894 INFO [RS:0;5d4d7471f19e:34103 {}] regionserver.HRegionServer(1031): Exiting; stopping=5d4d7471f19e,34103,1733555609616; zookeeper connection closed. 2024-12-07T07:13:37,894 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@251ddd8a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@251ddd8a 2024-12-07T07:13:37,895 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-07T07:13:37,915 DEBUG [M:0;5d4d7471f19e:46437 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5084ddc20ac348a6b74e83b4e3722331 is 69, key is 5d4d7471f19e,34103,1733555609616/rs:state/1733555610944/Put/seqid=0 2024-12-07T07:13:37,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741873_1049 (size=5442) 2024-12-07T07:13:37,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741873_1049 (size=5442) 2024-12-07T07:13:37,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741873_1049 (size=5442) 2024-12-07T07:13:37,922 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5084ddc20ac348a6b74e83b4e3722331 2024-12-07T07:13:37,929 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5084ddc20ac348a6b74e83b4e3722331 2024-12-07T07:13:37,943 DEBUG [M:0;5d4d7471f19e:46437 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/588a1efeea3443b5afbb61a370e2aadc is 52, key is load_balancer_on/state:d/1733555614435/Put/seqid=0 2024-12-07T07:13:37,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741874_1050 (size=5058) 2024-12-07T07:13:37,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741874_1050 (size=5058) 2024-12-07T07:13:37,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741874_1050 (size=5058) 2024-12-07T07:13:37,950 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/588a1efeea3443b5afbb61a370e2aadc 2024-12-07T07:13:37,956 DEBUG [M:0;5d4d7471f19e:46437 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/858e2b2150934a65b06f79ca0a1cfe6d as hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/858e2b2150934a65b06f79ca0a1cfe6d 2024-12-07T07:13:37,962 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/858e2b2150934a65b06f79ca0a1cfe6d, entries=14, sequenceid=575, filesize=5.9 K 2024-12-07T07:13:37,964 DEBUG [M:0;5d4d7471f19e:46437 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/554dfdc37dac4bb5a66df6ba5592efc9 as hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/554dfdc37dac4bb5a66df6ba5592efc9 2024-12-07T07:13:37,970 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/554dfdc37dac4bb5a66df6ba5592efc9, entries=79, sequenceid=575, filesize=12.5 K 2024-12-07T07:13:37,972 DEBUG [M:0;5d4d7471f19e:46437 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5084ddc20ac348a6b74e83b4e3722331 as hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5084ddc20ac348a6b74e83b4e3722331 2024-12-07T07:13:37,978 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5084ddc20ac348a6b74e83b4e3722331 2024-12-07T07:13:37,978 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5084ddc20ac348a6b74e83b4e3722331, entries=3, sequenceid=575, filesize=5.3 K 2024-12-07T07:13:37,979 DEBUG [M:0;5d4d7471f19e:46437 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/588a1efeea3443b5afbb61a370e2aadc as hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/588a1efeea3443b5afbb61a370e2aadc 2024-12-07T07:13:37,985 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43061/user/jenkins/test-data/47ef9e91-4c0d-e6c9-5cfc-b99d4e48e658/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/588a1efeea3443b5afbb61a370e2aadc, entries=1, sequenceid=575, filesize=4.9 K 2024-12-07T07:13:37,986 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.HRegion(3140): Finished flush of dataSize ~250.07 KB/256072, heapSize ~303.86 KB/311152, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 145ms, sequenceid=575, compaction requested=false 2024-12-07T07:13:37,988 INFO [M:0;5d4d7471f19e:46437 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T07:13:37,988 DEBUG [M:0;5d4d7471f19e:46437 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733555617841Disabling compacts and flushes for region at 1733555617841Disabling writes for close at 1733555617841Obtaining lock to block concurrent updates at 1733555617841Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733555617841Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=256072, getHeapSize=311152, getOffHeapSize=0, getCellsCount=714 at 1733555617842 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733555617842Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733555617842Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733555617855 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733555617855Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733555617868 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733555617885 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733555617885Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733555617900 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733555617914 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733555617914Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733555617929 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733555617942 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733555617942Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26b4e95a: reopening flushed file at 1733555617955 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@364dd0cb: reopening flushed file at 1733555617963 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a455ec0: reopening flushed file at 1733555617971 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b6b1483: reopening flushed file at 1733555617978 (+7 ms)Finished flush of dataSize ~250.07 KB/256072, heapSize ~303.86 KB/311152, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 145ms, sequenceid=575, compaction requested=false at 1733555617986 (+8 ms)Writing region close event to WAL at 1733555617988 (+2 ms)Closed at 1733555617988 2024-12-07T07:13:37,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38133 is added to blk_1073741830_1006 (size=294456) 2024-12-07T07:13:37,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41211 is added to blk_1073741830_1006 (size=294456) 2024-12-07T07:13:37,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44427 is added to blk_1073741830_1006 (size=294456) 2024-12-07T07:13:37,991 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T07:13:37,991 INFO [M:0;5d4d7471f19e:46437 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T07:13:37,991 INFO [M:0;5d4d7471f19e:46437 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46437 2024-12-07T07:13:37,992 INFO [M:0;5d4d7471f19e:46437 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T07:13:38,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T07:13:38,104 INFO [M:0;5d4d7471f19e:46437 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T07:13:38,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46437-0x101aea355730000, quorum=127.0.0.1:63017, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T07:13:38,146 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@78e5b4c2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T07:13:38,148 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@687f641d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T07:13:38,149 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T07:13:38,149 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18213c95{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T07:13:38,149 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76da50c3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/hadoop.log.dir/,STOPPED} 2024-12-07T07:13:38,151 WARN [BP-125736803-172.17.0.2-1733555604035 heartbeating to localhost/127.0.0.1:43061 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T07:13:38,151 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-07T07:13:38,151 WARN [BP-125736803-172.17.0.2-1733555604035 heartbeating to localhost/127.0.0.1:43061 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-125736803-172.17.0.2-1733555604035 (Datanode Uuid 02bb7af9-b95d-49e2-ad84-77bc082e7c0c) service to localhost/127.0.0.1:43061 2024-12-07T07:13:38,151 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T07:13:38,152 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data5/current/BP-125736803-172.17.0.2-1733555604035 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T07:13:38,152 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data6/current/BP-125736803-172.17.0.2-1733555604035 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T07:13:38,153 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T07:13:38,155 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@14ce6d63{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T07:13:38,155 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@101b9d11{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T07:13:38,155 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T07:13:38,155 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7913951{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T07:13:38,155 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7e78dc0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/hadoop.log.dir/,STOPPED} 2024-12-07T07:13:38,156 WARN [BP-125736803-172.17.0.2-1733555604035 heartbeating to localhost/127.0.0.1:43061 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T07:13:38,156 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-07T07:13:38,156 WARN [BP-125736803-172.17.0.2-1733555604035 heartbeating to localhost/127.0.0.1:43061 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-125736803-172.17.0.2-1733555604035 (Datanode Uuid 797ecaaa-a207-4210-908c-38d4a465da06) service to localhost/127.0.0.1:43061 2024-12-07T07:13:38,156 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T07:13:38,157 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data3/current/BP-125736803-172.17.0.2-1733555604035 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T07:13:38,157 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data4/current/BP-125736803-172.17.0.2-1733555604035 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T07:13:38,157 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T07:13:38,159 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@649cead0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T07:13:38,159 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@742290fe{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T07:13:38,159 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T07:13:38,160 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7edb5709{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T07:13:38,160 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7de7e807{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/hadoop.log.dir/,STOPPED} 2024-12-07T07:13:38,161 WARN [BP-125736803-172.17.0.2-1733555604035 heartbeating to localhost/127.0.0.1:43061 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T07:13:38,161 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-07T07:13:38,161 WARN [BP-125736803-172.17.0.2-1733555604035 heartbeating to localhost/127.0.0.1:43061 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-125736803-172.17.0.2-1733555604035 (Datanode Uuid 19740c13-1a4a-4982-a7f4-3a67004d3309) service to localhost/127.0.0.1:43061 2024-12-07T07:13:38,161 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T07:13:38,161 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data1/current/BP-125736803-172.17.0.2-1733555604035 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T07:13:38,161 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/cluster_0ff71a1c-97ae-518c-5f20-283a794b7422/data/data2/current/BP-125736803-172.17.0.2-1733555604035 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T07:13:38,162 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T07:13:38,169 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@302cfe51{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-07T07:13:38,170 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c1eb5a9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T07:13:38,170 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T07:13:38,170 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@378c34a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T07:13:38,170 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@952689a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b7f7fd7-df26-8758-788f-d1886f9ca3a0/hadoop.log.dir/,STOPPED} 2024-12-07T07:13:38,180 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T07:13:38,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down