2024-11-26 19:14:24,578 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-26 19:14:24,652 main DEBUG Took 0.070054 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 19:14:24,657 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 19:14:24,658 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 19:14:24,659 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 19:14:24,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,693 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 19:14:24,726 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,752 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,758 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,759 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,759 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,760 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,761 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,762 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,762 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,763 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,764 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,764 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,770 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,771 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,772 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,773 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,774 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,774 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,775 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,777 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,778 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,780 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 19:14:24,785 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,785 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 19:14:24,788 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 19:14:24,792 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 19:14:24,804 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 19:14:24,805 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 19:14:24,811 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 19:14:24,812 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 19:14:24,825 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 19:14:24,831 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 19:14:24,833 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 19:14:24,834 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 19:14:24,835 main DEBUG createAppenders(={Console}) 2024-11-26 19:14:24,836 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-26 19:14:24,838 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-26 19:14:24,838 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-26 19:14:24,839 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 19:14:24,839 main DEBUG OutputStream closed 2024-11-26 19:14:24,840 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 19:14:24,840 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 19:14:24,841 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-26 19:14:24,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 19:14:24,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 19:14:24,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 19:14:24,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 19:14:24,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 19:14:24,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 19:14:24,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 19:14:24,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 19:14:24,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 19:14:24,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 19:14:24,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 19:14:24,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 19:14:24,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 19:14:24,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 19:14:24,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 19:14:24,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 19:14:24,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 19:14:24,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 19:14:24,959 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 19:14:24,960 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-26 19:14:24,961 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 19:14:24,962 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-26T19:14:24,980 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-26 19:14:24,984 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 19:14:24,984 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T19:14:25,365 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce 2024-11-26T19:14:25,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T19:14:25,444 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f, deleteOnExit=true 2024-11-26T19:14:25,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T19:14:25,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/test.cache.data in system properties and HBase conf 2024-11-26T19:14:25,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T19:14:25,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/hadoop.log.dir in system properties and HBase conf 2024-11-26T19:14:25,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T19:14:25,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T19:14:25,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T19:14:25,600 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T19:14:25,753 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T19:14:25,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T19:14:25,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T19:14:25,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T19:14:25,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T19:14:25,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T19:14:25,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T19:14:25,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T19:14:25,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T19:14:25,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T19:14:25,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/nfs.dump.dir in system properties and HBase conf 2024-11-26T19:14:25,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/java.io.tmpdir in system properties and HBase conf 2024-11-26T19:14:25,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T19:14:25,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T19:14:25,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T19:14:26,924 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T19:14:27,037 INFO [Time-limited test {}] log.Log(170): Logging initialized @3550ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T19:14:27,135 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T19:14:27,216 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T19:14:27,253 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T19:14:27,253 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T19:14:27,255 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T19:14:27,272 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T19:14:27,279 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/hadoop.log.dir/,AVAILABLE} 2024-11-26T19:14:27,281 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T19:14:27,548 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/java.io.tmpdir/jetty-localhost-42029-hadoop-hdfs-3_4_1-tests_jar-_-any-5552752320987946339/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T19:14:27,556 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:42029} 2024-11-26T19:14:27,557 INFO [Time-limited test {}] server.Server(415): Started @4072ms 2024-11-26T19:14:28,027 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T19:14:28,036 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T19:14:28,041 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T19:14:28,041 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T19:14:28,041 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T19:14:28,042 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51c39823{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/hadoop.log.dir/,AVAILABLE} 2024-11-26T19:14:28,043 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c7135d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T19:14:28,212 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65e17246{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/java.io.tmpdir/jetty-localhost-41587-hadoop-hdfs-3_4_1-tests_jar-_-any-18302364258626149895/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T19:14:28,213 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@220c8d80{HTTP/1.1, (http/1.1)}{localhost:41587} 2024-11-26T19:14:28,213 INFO [Time-limited test {}] server.Server(415): Started @4728ms 2024-11-26T19:14:28,280 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T19:14:28,461 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T19:14:28,470 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T19:14:28,473 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T19:14:28,473 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T19:14:28,473 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T19:14:28,481 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ff36c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/hadoop.log.dir/,AVAILABLE} 2024-11-26T19:14:28,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6331ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T19:14:28,664 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@395ff33a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/java.io.tmpdir/jetty-localhost-40413-hadoop-hdfs-3_4_1-tests_jar-_-any-17989317309961691382/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T19:14:28,667 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d24ac65{HTTP/1.1, (http/1.1)}{localhost:40413} 2024-11-26T19:14:28,667 INFO [Time-limited test {}] server.Server(415): Started @5183ms 2024-11-26T19:14:28,670 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T19:14:28,794 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T19:14:28,803 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T19:14:28,818 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T19:14:28,819 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T19:14:28,819 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T19:14:28,825 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ed43cd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/hadoop.log.dir/,AVAILABLE} 2024-11-26T19:14:28,826 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48a2d1c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T19:14:28,942 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data3/current/BP-189323584-172.17.0.2-1732648466610/current, will proceed with Du for space computation calculation, 2024-11-26T19:14:28,942 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data1/current/BP-189323584-172.17.0.2-1732648466610/current, will proceed with Du for space computation calculation, 2024-11-26T19:14:28,949 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data2/current/BP-189323584-172.17.0.2-1732648466610/current, will proceed with Du for space computation calculation, 2024-11-26T19:14:28,958 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data4/current/BP-189323584-172.17.0.2-1732648466610/current, will proceed with Du for space computation calculation, 2024-11-26T19:14:28,999 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@46041a9d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/java.io.tmpdir/jetty-localhost-38197-hadoop-hdfs-3_4_1-tests_jar-_-any-11295102392964791729/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T19:14:29,001 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a39ea25{HTTP/1.1, (http/1.1)}{localhost:38197} 2024-11-26T19:14:29,001 INFO [Time-limited test {}] server.Server(415): Started @5516ms 2024-11-26T19:14:29,004 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T19:14:29,025 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T19:14:29,056 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T19:14:29,138 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1b74ddb804df260c with lease ID 0x6d94ca40bc6785da: Processing first storage report for DS-79d5558f-903b-428b-9867-cd303d28a575 from datanode DatanodeRegistration(127.0.0.1:39887, datanodeUuid=e17e6176-ff11-454f-b067-a0bc85f8b859, infoPort=33867, infoSecurePort=0, ipcPort=34965, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610) 2024-11-26T19:14:29,139 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1b74ddb804df260c with lease ID 0x6d94ca40bc6785da: from storage DS-79d5558f-903b-428b-9867-cd303d28a575 node DatanodeRegistration(127.0.0.1:39887, datanodeUuid=e17e6176-ff11-454f-b067-a0bc85f8b859, infoPort=33867, infoSecurePort=0, ipcPort=34965, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-26T19:14:29,140 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x434ac092ecf6614e with lease ID 0x6d94ca40bc6785db: Processing first storage report for DS-b118e9ce-ffe5-4017-9420-f930aa963eec from datanode DatanodeRegistration(127.0.0.1:37015, datanodeUuid=7eb2bab3-e40a-4ec6-814c-8891a0b06d7b, infoPort=43795, infoSecurePort=0, ipcPort=40983, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610) 2024-11-26T19:14:29,140 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x434ac092ecf6614e with lease ID 0x6d94ca40bc6785db: from storage DS-b118e9ce-ffe5-4017-9420-f930aa963eec node DatanodeRegistration(127.0.0.1:37015, datanodeUuid=7eb2bab3-e40a-4ec6-814c-8891a0b06d7b, infoPort=43795, infoSecurePort=0, ipcPort=40983, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T19:14:29,140 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1b74ddb804df260c with lease ID 0x6d94ca40bc6785da: Processing first storage report for DS-7c69486f-0d06-4075-ab57-ed1242f5fb44 from datanode DatanodeRegistration(127.0.0.1:39887, datanodeUuid=e17e6176-ff11-454f-b067-a0bc85f8b859, infoPort=33867, infoSecurePort=0, ipcPort=34965, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610) 2024-11-26T19:14:29,141 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1b74ddb804df260c with lease ID 0x6d94ca40bc6785da: from storage DS-7c69486f-0d06-4075-ab57-ed1242f5fb44 node DatanodeRegistration(127.0.0.1:39887, datanodeUuid=e17e6176-ff11-454f-b067-a0bc85f8b859, infoPort=33867, infoSecurePort=0, ipcPort=34965, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T19:14:29,141 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x434ac092ecf6614e with lease ID 0x6d94ca40bc6785db: Processing first storage report for DS-2ca6e115-eb1d-4e6c-be4a-a9e46d5a2113 from datanode DatanodeRegistration(127.0.0.1:37015, datanodeUuid=7eb2bab3-e40a-4ec6-814c-8891a0b06d7b, infoPort=43795, infoSecurePort=0, ipcPort=40983, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610) 2024-11-26T19:14:29,141 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x434ac092ecf6614e with lease ID 0x6d94ca40bc6785db: from storage DS-2ca6e115-eb1d-4e6c-be4a-a9e46d5a2113 node DatanodeRegistration(127.0.0.1:37015, datanodeUuid=7eb2bab3-e40a-4ec6-814c-8891a0b06d7b, infoPort=43795, infoSecurePort=0, ipcPort=40983, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T19:14:29,281 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data5/current/BP-189323584-172.17.0.2-1732648466610/current, will proceed with Du for space computation calculation, 2024-11-26T19:14:29,289 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data6/current/BP-189323584-172.17.0.2-1732648466610/current, will proceed with Du for space computation calculation, 2024-11-26T19:14:29,393 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T19:14:29,406 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd73d701a25c159e6 with lease ID 0x6d94ca40bc6785dc: Processing first storage report for DS-d8f8d6e7-0acd-480c-8a0f-0732624c813b from datanode DatanodeRegistration(127.0.0.1:38815, datanodeUuid=e521ff40-7d72-4f96-bd9d-fbacdf0eee14, infoPort=36327, infoSecurePort=0, ipcPort=39679, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610) 2024-11-26T19:14:29,406 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd73d701a25c159e6 with lease ID 0x6d94ca40bc6785dc: from storage DS-d8f8d6e7-0acd-480c-8a0f-0732624c813b node DatanodeRegistration(127.0.0.1:38815, datanodeUuid=e521ff40-7d72-4f96-bd9d-fbacdf0eee14, infoPort=36327, infoSecurePort=0, ipcPort=39679, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T19:14:29,406 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd73d701a25c159e6 with lease ID 0x6d94ca40bc6785dc: Processing first storage report for DS-9cdaaf8c-eaf0-4d0f-8eb9-323baed007e8 from datanode DatanodeRegistration(127.0.0.1:38815, datanodeUuid=e521ff40-7d72-4f96-bd9d-fbacdf0eee14, infoPort=36327, infoSecurePort=0, ipcPort=39679, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610) 2024-11-26T19:14:29,407 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd73d701a25c159e6 with lease ID 0x6d94ca40bc6785dc: from storage DS-9cdaaf8c-eaf0-4d0f-8eb9-323baed007e8 node DatanodeRegistration(127.0.0.1:38815, datanodeUuid=e521ff40-7d72-4f96-bd9d-fbacdf0eee14, infoPort=36327, infoSecurePort=0, ipcPort=39679, storageInfo=lv=-57;cid=testClusterID;nsid=974250810;c=1732648466610), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T19:14:29,599 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce 2024-11-26T19:14:29,721 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/zookeeper_0, clientPort=56139, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T19:14:29,734 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56139 2024-11-26T19:14:29,746 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:29,750 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:30,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741825_1001 (size=7) 2024-11-26T19:14:30,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741825_1001 (size=7) 2024-11-26T19:14:30,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741825_1001 (size=7) 2024-11-26T19:14:30,488 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e with version=8 2024-11-26T19:14:30,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/hbase-staging 2024-11-26T19:14:30,592 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T19:14:30,906 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fe76acf732b2:0 server-side Connection retries=45 2024-11-26T19:14:30,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:30,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:30,924 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T19:14:30,924 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:30,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T19:14:31,097 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T19:14:31,170 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T19:14:31,181 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T19:14:31,187 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T19:14:31,225 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7774 (auto-detected) 2024-11-26T19:14:31,227 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T19:14:31,260 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39807 2024-11-26T19:14:31,292 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39807 connecting to ZooKeeper ensemble=127.0.0.1:56139 2024-11-26T19:14:31,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:398070x0, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T19:14:31,346 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39807-0x10155b36f3b0000 connected 2024-11-26T19:14:31,387 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:31,400 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:31,429 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:14:31,435 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e, hbase.cluster.distributed=false 2024-11-26T19:14:31,491 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T19:14:31,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39807 2024-11-26T19:14:31,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39807 2024-11-26T19:14:31,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39807 2024-11-26T19:14:31,515 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39807 2024-11-26T19:14:31,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39807 2024-11-26T19:14:31,653 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fe76acf732b2:0 server-side Connection retries=45 2024-11-26T19:14:31,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:31,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:31,655 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T19:14:31,656 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:31,656 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T19:14:31,659 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T19:14:31,661 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T19:14:31,662 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35951 2024-11-26T19:14:31,664 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35951 connecting to ZooKeeper ensemble=127.0.0.1:56139 2024-11-26T19:14:31,666 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:31,670 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:31,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359510x0, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T19:14:31,678 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35951-0x10155b36f3b0001 connected 2024-11-26T19:14:31,681 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:14:31,686 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T19:14:31,695 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T19:14:31,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T19:14:31,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T19:14:31,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35951 2024-11-26T19:14:31,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35951 2024-11-26T19:14:31,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35951 2024-11-26T19:14:31,715 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35951 2024-11-26T19:14:31,716 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35951 2024-11-26T19:14:31,742 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fe76acf732b2:0 server-side Connection retries=45 2024-11-26T19:14:31,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:31,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:31,744 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T19:14:31,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:31,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T19:14:31,744 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T19:14:31,745 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T19:14:31,746 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39793 2024-11-26T19:14:31,748 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39793 connecting to ZooKeeper ensemble=127.0.0.1:56139 2024-11-26T19:14:31,750 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:31,753 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:31,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:397930x0, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T19:14:31,766 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:397930x0, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:14:31,767 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T19:14:31,767 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39793-0x10155b36f3b0002 connected 2024-11-26T19:14:31,771 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T19:14:31,775 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T19:14:31,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T19:14:31,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39793 2024-11-26T19:14:31,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39793 2024-11-26T19:14:31,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39793 2024-11-26T19:14:31,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39793 2024-11-26T19:14:31,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39793 2024-11-26T19:14:31,828 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fe76acf732b2:0 server-side Connection retries=45 2024-11-26T19:14:31,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:31,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:31,829 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T19:14:31,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T19:14:31,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T19:14:31,829 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T19:14:31,830 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T19:14:31,832 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37087 2024-11-26T19:14:31,834 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37087 connecting to ZooKeeper ensemble=127.0.0.1:56139 2024-11-26T19:14:31,835 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:31,838 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:31,854 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:370870x0, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:14:31,855 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T19:14:31,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370870x0, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T19:14:31,857 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T19:14:31,858 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:370870x0, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T19:14:31,861 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37087-0x10155b36f3b0003 connected 2024-11-26T19:14:31,862 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T19:14:31,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37087 2024-11-26T19:14:31,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37087 2024-11-26T19:14:31,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37087 2024-11-26T19:14:31,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37087 2024-11-26T19:14:31,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37087 2024-11-26T19:14:31,903 DEBUG [M:0;fe76acf732b2:39807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fe76acf732b2:39807 2024-11-26T19:14:31,904 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fe76acf732b2,39807,1732648470659 2024-11-26T19:14:31,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:14:31,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:14:31,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:14:31,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:14:31,917 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fe76acf732b2,39807,1732648470659 2024-11-26T19:14:31,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:31,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T19:14:31,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:31,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T19:14:31,955 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T19:14:31,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T19:14:31,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:31,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:31,960 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fe76acf732b2,39807,1732648470659 from backup master directory 2024-11-26T19:14:31,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:14:31,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:14:31,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:14:31,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fe76acf732b2,39807,1732648470659 2024-11-26T19:14:31,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T19:14:31,965 WARN [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T19:14:31,966 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fe76acf732b2,39807,1732648470659 2024-11-26T19:14:31,969 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T19:14:31,970 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T19:14:32,039 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/hbase.id] with ID: 4344240a-10ff-4a1f-a2a7-857ca6573a99 2024-11-26T19:14:32,039 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/.tmp/hbase.id 2024-11-26T19:14:32,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741826_1002 (size=42) 2024-11-26T19:14:32,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741826_1002 (size=42) 2024-11-26T19:14:32,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741826_1002 (size=42) 2024-11-26T19:14:32,059 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/.tmp/hbase.id]:[hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/hbase.id] 2024-11-26T19:14:32,115 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T19:14:32,121 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T19:14:32,146 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-26T19:14:32,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741827_1003 (size=196) 2024-11-26T19:14:32,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741827_1003 (size=196) 2024-11-26T19:14:32,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741827_1003 (size=196) 2024-11-26T19:14:32,204 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:14:32,207 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T19:14:32,228 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:32,233 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T19:14:32,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741828_1004 (size=1189) 2024-11-26T19:14:32,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741828_1004 (size=1189) 2024-11-26T19:14:32,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741828_1004 (size=1189) 2024-11-26T19:14:32,308 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store 2024-11-26T19:14:32,331 WARN [IPC Server handler 0 on default port 45145 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T19:14:32,331 WARN [IPC Server handler 0 on default port 45145 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T19:14:32,332 WARN [IPC Server handler 0 on default port 45145 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T19:14:32,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741829_1005 (size=34) 2024-11-26T19:14:32,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741829_1005 (size=34) 2024-11-26T19:14:32,361 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T19:14:32,366 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:32,367 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T19:14:32,367 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:14:32,368 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:14:32,369 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T19:14:32,370 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:14:32,370 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:14:32,371 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732648472367Disabling compacts and flushes for region at 1732648472367Disabling writes for close at 1732648472369 (+2 ms)Writing region close event to WAL at 1732648472370 (+1 ms)Closed at 1732648472370 2024-11-26T19:14:32,374 WARN [master/fe76acf732b2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/.initializing 2024-11-26T19:14:32,374 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/WALs/fe76acf732b2,39807,1732648470659 2024-11-26T19:14:32,385 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T19:14:32,404 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fe76acf732b2%2C39807%2C1732648470659, suffix=, logDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/WALs/fe76acf732b2,39807,1732648470659, archiveDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/oldWALs, maxLogs=10 2024-11-26T19:14:32,438 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/WALs/fe76acf732b2,39807,1732648470659/fe76acf732b2%2C39807%2C1732648470659.1732648472411, exclude list is [], retry=0 2024-11-26T19:14:32,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37015,DS-b118e9ce-ffe5-4017-9420-f930aa963eec,DISK] 2024-11-26T19:14:32,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39887,DS-79d5558f-903b-428b-9867-cd303d28a575,DISK] 2024-11-26T19:14:32,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38815,DS-d8f8d6e7-0acd-480c-8a0f-0732624c813b,DISK] 2024-11-26T19:14:32,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T19:14:32,523 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/WALs/fe76acf732b2,39807,1732648470659/fe76acf732b2%2C39807%2C1732648470659.1732648472411 2024-11-26T19:14:32,525 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43795:43795),(127.0.0.1/127.0.0.1:36327:36327),(127.0.0.1/127.0.0.1:33867:33867)] 2024-11-26T19:14:32,526 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:14:32,527 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:32,532 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,533 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T19:14:32,637 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:32,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:32,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,647 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T19:14:32,648 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:32,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:32,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T19:14:32,656 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:32,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:32,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T19:14:32,664 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:32,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:32,666 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,672 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,674 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,683 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,684 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,688 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T19:14:32,695 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T19:14:32,710 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:32,711 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58946907, jitterRate=-0.12162263691425323}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T19:14:32,720 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732648472554Initializing all the Stores at 1732648472556 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648472557 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648472560 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648472561 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648472561Cleaning up temporary data from old regions at 1732648472684 (+123 ms)Region opened successfully at 1732648472720 (+36 ms) 2024-11-26T19:14:32,723 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T19:14:32,769 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ddbc507, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fe76acf732b2/172.17.0.2:0 2024-11-26T19:14:32,810 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T19:14:32,824 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:32,824 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:32,828 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:32,829 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T19:14:32,836 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-26T19:14:32,836 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T19:14:32,871 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T19:14:32,881 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T19:14:32,883 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T19:14:32,885 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T19:14:32,887 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T19:14:32,889 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T19:14:32,892 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T19:14:32,897 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T19:14:32,899 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T19:14:32,900 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T19:14:32,903 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T19:14:32,923 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T19:14:32,925 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T19:14:32,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T19:14:32,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T19:14:32,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T19:14:32,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T19:14:32,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,935 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fe76acf732b2,39807,1732648470659, sessionid=0x10155b36f3b0000, setting cluster-up flag (Was=false) 2024-11-26T19:14:32,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,960 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T19:14:32,962 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fe76acf732b2,39807,1732648470659 2024-11-26T19:14:32,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:32,979 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T19:14:32,981 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fe76acf732b2,39807,1732648470659 2024-11-26T19:14:32,987 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T19:14:33,068 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:33,081 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T19:14:33,085 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(746): ClusterId : 4344240a-10ff-4a1f-a2a7-857ca6573a99 2024-11-26T19:14:33,086 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(746): ClusterId : 4344240a-10ff-4a1f-a2a7-857ca6573a99 2024-11-26T19:14:33,086 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(746): ClusterId : 4344240a-10ff-4a1f-a2a7-857ca6573a99 2024-11-26T19:14:33,088 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T19:14:33,088 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T19:14:33,088 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T19:14:33,092 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T19:14:33,095 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T19:14:33,096 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T19:14:33,096 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T19:14:33,096 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T19:14:33,097 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T19:14:33,097 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T19:14:33,102 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T19:14:33,102 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T19:14:33,099 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fe76acf732b2,39807,1732648470659 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T19:14:33,103 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T19:14:33,103 DEBUG [RS:2;fe76acf732b2:37087 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c8a981, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fe76acf732b2/172.17.0.2:0 2024-11-26T19:14:33,105 DEBUG [RS:1;fe76acf732b2:39793 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de31050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fe76acf732b2/172.17.0.2:0 2024-11-26T19:14:33,119 DEBUG [RS:0;fe76acf732b2:35951 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3c60df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fe76acf732b2/172.17.0.2:0 2024-11-26T19:14:33,123 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fe76acf732b2:0, corePoolSize=5, maxPoolSize=5 2024-11-26T19:14:33,124 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fe76acf732b2:0, corePoolSize=5, maxPoolSize=5 2024-11-26T19:14:33,124 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fe76acf732b2:0, corePoolSize=5, maxPoolSize=5 2024-11-26T19:14:33,124 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fe76acf732b2:0, corePoolSize=5, maxPoolSize=5 2024-11-26T19:14:33,124 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fe76acf732b2:0, corePoolSize=10, maxPoolSize=10 2024-11-26T19:14:33,124 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,124 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fe76acf732b2:0, corePoolSize=2, maxPoolSize=2 2024-11-26T19:14:33,125 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,127 DEBUG [RS:1;fe76acf732b2:39793 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fe76acf732b2:39793 2024-11-26T19:14:33,129 DEBUG [RS:2;fe76acf732b2:37087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;fe76acf732b2:37087 2024-11-26T19:14:33,133 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T19:14:33,133 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T19:14:33,133 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T19:14:33,134 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T19:14:33,134 DEBUG [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T19:14:33,134 DEBUG [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T19:14:33,137 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(2659): reportForDuty to master=fe76acf732b2,39807,1732648470659 with port=37087, startcode=1732648471827 2024-11-26T19:14:33,137 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(2659): reportForDuty to master=fe76acf732b2,39807,1732648470659 with port=39793, startcode=1732648471742 2024-11-26T19:14:33,150 DEBUG [RS:0;fe76acf732b2:35951 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fe76acf732b2:35951 2024-11-26T19:14:33,150 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T19:14:33,150 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T19:14:33,150 DEBUG [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T19:14:33,151 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(2659): reportForDuty to master=fe76acf732b2,39807,1732648470659 with port=35951, startcode=1732648471608 2024-11-26T19:14:33,152 DEBUG [RS:0;fe76acf732b2:35951 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T19:14:33,152 DEBUG [RS:1;fe76acf732b2:39793 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T19:14:33,156 DEBUG [RS:2;fe76acf732b2:37087 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T19:14:33,166 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648503166 2024-11-26T19:14:33,168 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T19:14:33,169 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T19:14:33,175 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T19:14:33,175 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T19:14:33,175 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T19:14:33,175 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T19:14:33,204 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:33,204 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T19:14:33,212 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:33,212 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T19:14:33,214 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,249 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T19:14:33,250 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T19:14:33,251 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T19:14:33,262 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53973, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T19:14:33,263 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T19:14:33,264 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T19:14:33,266 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41595, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T19:14:33,268 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42025, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T19:14:33,269 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:33,278 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:33,280 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:33,288 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.large.0-1732648473265,5,FailOnTimeoutGroup] 2024-11-26T19:14:33,290 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.small.0-1732648473289,5,FailOnTimeoutGroup] 2024-11-26T19:14:33,291 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,291 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T19:14:33,314 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,314 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741831_1007 (size=1321) 2024-11-26T19:14:33,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741831_1007 (size=1321) 2024-11-26T19:14:33,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741831_1007 (size=1321) 2024-11-26T19:14:33,324 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T19:14:33,324 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:33,329 DEBUG [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T19:14:33,329 DEBUG [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T19:14:33,329 DEBUG [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T19:14:33,329 WARN [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T19:14:33,329 WARN [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T19:14:33,329 WARN [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T19:14:33,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741832_1008 (size=32) 2024-11-26T19:14:33,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741832_1008 (size=32) 2024-11-26T19:14:33,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741832_1008 (size=32) 2024-11-26T19:14:33,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:33,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T19:14:33,353 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T19:14:33,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:33,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:33,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T19:14:33,362 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T19:14:33,362 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:33,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:33,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T19:14:33,367 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T19:14:33,367 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:33,368 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:33,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T19:14:33,372 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T19:14:33,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:33,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:33,373 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T19:14:33,375 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740 2024-11-26T19:14:33,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740 2024-11-26T19:14:33,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T19:14:33,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T19:14:33,382 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T19:14:33,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T19:14:33,396 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:33,397 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60287091, jitterRate=-0.10165233910083771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T19:14:33,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732648473348Initializing all the Stores at 1732648473350 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648473350Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648473350Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648473350Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648473350Cleaning up temporary data from old regions at 1732648473381 (+31 ms)Region opened successfully at 1732648473400 (+19 ms) 2024-11-26T19:14:33,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T19:14:33,400 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T19:14:33,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T19:14:33,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T19:14:33,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T19:14:33,402 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T19:14:33,402 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732648473400Disabling compacts and flushes for region at 1732648473400Disabling writes for close at 1732648473400Writing region close event to WAL at 1732648473401 (+1 ms)Closed at 1732648473402 (+1 ms) 2024-11-26T19:14:33,413 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:33,413 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T19:14:33,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T19:14:33,431 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(2659): reportForDuty to master=fe76acf732b2,39807,1732648470659 with port=39793, startcode=1732648471742 2024-11-26T19:14:33,432 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(2659): reportForDuty to master=fe76acf732b2,39807,1732648470659 with port=37087, startcode=1732648471827 2024-11-26T19:14:33,433 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(2659): reportForDuty to master=fe76acf732b2,39807,1732648470659 with port=35951, startcode=1732648471608 2024-11-26T19:14:33,433 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fe76acf732b2,39793,1732648471742 2024-11-26T19:14:33,436 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] master.ServerManager(517): Registering regionserver=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:33,449 DEBUG [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:33,449 DEBUG [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45145 2024-11-26T19:14:33,449 DEBUG [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T19:14:33,449 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:33,453 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T19:14:33,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T19:14:33,456 DEBUG [RS:1;fe76acf732b2:39793 {}] zookeeper.ZKUtil(111): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fe76acf732b2,39793,1732648471742 2024-11-26T19:14:33,456 WARN [RS:1;fe76acf732b2:39793 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T19:14:33,456 INFO [RS:1;fe76acf732b2:39793 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T19:14:33,456 DEBUG [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742 2024-11-26T19:14:33,457 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fe76acf732b2,37087,1732648471827 2024-11-26T19:14:33,458 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] master.ServerManager(517): Registering regionserver=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:33,463 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fe76acf732b2,35951,1732648471608 2024-11-26T19:14:33,463 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] master.ServerManager(517): Registering regionserver=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:33,465 DEBUG [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:33,465 DEBUG [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45145 2024-11-26T19:14:33,465 DEBUG [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T19:14:33,465 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fe76acf732b2,39793,1732648471742] 2024-11-26T19:14:33,468 DEBUG [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:33,469 DEBUG [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45145 2024-11-26T19:14:33,469 DEBUG [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T19:14:33,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T19:14:33,476 DEBUG [RS:2;fe76acf732b2:37087 {}] zookeeper.ZKUtil(111): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fe76acf732b2,37087,1732648471827 2024-11-26T19:14:33,476 WARN [RS:2;fe76acf732b2:37087 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T19:14:33,477 INFO [RS:2;fe76acf732b2:37087 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T19:14:33,477 DEBUG [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,37087,1732648471827 2024-11-26T19:14:33,478 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fe76acf732b2,37087,1732648471827] 2024-11-26T19:14:33,478 DEBUG [RS:0;fe76acf732b2:35951 {}] zookeeper.ZKUtil(111): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fe76acf732b2,35951,1732648471608 2024-11-26T19:14:33,478 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fe76acf732b2,35951,1732648471608] 2024-11-26T19:14:33,478 WARN [RS:0;fe76acf732b2:35951 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T19:14:33,479 INFO [RS:0;fe76acf732b2:35951 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T19:14:33,479 DEBUG [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,35951,1732648471608 2024-11-26T19:14:33,517 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T19:14:33,517 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T19:14:33,517 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T19:14:33,544 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T19:14:33,544 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T19:14:33,548 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T19:14:33,562 INFO [RS:2;fe76acf732b2:37087 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T19:14:33,563 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,568 INFO [RS:0;fe76acf732b2:35951 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T19:14:33,569 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,569 INFO [RS:1;fe76acf732b2:39793 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T19:14:33,569 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,570 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T19:14:33,572 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T19:14:33,576 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T19:14:33,579 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T19:14:33,579 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T19:14:33,580 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T19:14:33,581 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,581 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,581 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,581 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,581 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,581 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,581 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,581 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,581 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,581 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,581 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,582 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,582 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0, corePoolSize=2, maxPoolSize=2 2024-11-26T19:14:33,582 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0, corePoolSize=2, maxPoolSize=2 2024-11-26T19:14:33,582 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,582 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,582 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,582 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,582 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,582 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,582 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,582 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,582 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,583 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,583 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,583 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,583 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fe76acf732b2:0, corePoolSize=3, maxPoolSize=3 2024-11-26T19:14:33,583 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fe76acf732b2:0, corePoolSize=3, maxPoolSize=3 2024-11-26T19:14:33,583 DEBUG [RS:0;fe76acf732b2:35951 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0, corePoolSize=3, maxPoolSize=3 2024-11-26T19:14:33,583 DEBUG [RS:1;fe76acf732b2:39793 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0, corePoolSize=3, maxPoolSize=3 2024-11-26T19:14:33,584 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,585 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,585 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,585 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,585 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,586 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,586 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0, corePoolSize=2, maxPoolSize=2 2024-11-26T19:14:33,586 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,586 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,586 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,586 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,587 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,587 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fe76acf732b2:0, corePoolSize=1, maxPoolSize=1 2024-11-26T19:14:33,587 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fe76acf732b2:0, corePoolSize=3, maxPoolSize=3 2024-11-26T19:14:33,587 DEBUG [RS:2;fe76acf732b2:37087 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fe76acf732b2:0, corePoolSize=3, maxPoolSize=3 2024-11-26T19:14:33,599 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,599 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,599 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,600 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,600 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,600 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,35951,1732648471608-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T19:14:33,600 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,600 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,600 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,601 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,601 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,601 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,39793,1732648471742-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T19:14:33,602 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,603 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,603 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,603 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,603 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,603 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,37087,1732648471827-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T19:14:33,604 WARN [fe76acf732b2:39807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T19:14:33,653 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T19:14:33,654 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T19:14:33,656 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,35951,1732648471608-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,656 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,653 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T19:14:33,657 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.Replication(171): fe76acf732b2,35951,1732648471608 started 2024-11-26T19:14:33,657 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,37087,1732648471827-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,657 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,657 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.Replication(171): fe76acf732b2,37087,1732648471827 started 2024-11-26T19:14:33,658 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,39793,1732648471742-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,662 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,663 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.Replication(171): fe76acf732b2,39793,1732648471742 started 2024-11-26T19:14:33,691 INFO [RS:1;fe76acf732b2:39793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,692 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(1482): Serving as fe76acf732b2,39793,1732648471742, RpcServer on fe76acf732b2/172.17.0.2:39793, sessionid=0x10155b36f3b0002 2024-11-26T19:14:33,693 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,694 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T19:14:33,694 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(1482): Serving as fe76acf732b2,37087,1732648471827, RpcServer on fe76acf732b2/172.17.0.2:37087, sessionid=0x10155b36f3b0003 2024-11-26T19:14:33,694 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T19:14:33,694 DEBUG [RS:2;fe76acf732b2:37087 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fe76acf732b2,37087,1732648471827 2024-11-26T19:14:33,694 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fe76acf732b2,37087,1732648471827' 2024-11-26T19:14:33,694 DEBUG [RS:1;fe76acf732b2:39793 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fe76acf732b2,39793,1732648471742 2024-11-26T19:14:33,694 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fe76acf732b2,39793,1732648471742' 2024-11-26T19:14:33,694 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T19:14:33,695 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T19:14:33,696 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T19:14:33,698 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T19:14:33,699 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T19:14:33,699 DEBUG [RS:1;fe76acf732b2:39793 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fe76acf732b2,39793,1732648471742 2024-11-26T19:14:33,699 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:33,699 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(1482): Serving as fe76acf732b2,35951,1732648471608, RpcServer on fe76acf732b2/172.17.0.2:35951, sessionid=0x10155b36f3b0001 2024-11-26T19:14:33,700 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T19:14:33,700 DEBUG [RS:0;fe76acf732b2:35951 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fe76acf732b2,35951,1732648471608 2024-11-26T19:14:33,700 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fe76acf732b2,35951,1732648471608' 2024-11-26T19:14:33,700 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T19:14:33,701 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fe76acf732b2,39793,1732648471742' 2024-11-26T19:14:33,701 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T19:14:33,701 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T19:14:33,713 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T19:14:33,713 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T19:14:33,713 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T19:14:33,713 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T19:14:33,713 DEBUG [RS:2;fe76acf732b2:37087 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fe76acf732b2,37087,1732648471827 2024-11-26T19:14:33,714 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fe76acf732b2,37087,1732648471827' 2024-11-26T19:14:33,714 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T19:14:33,714 DEBUG [RS:1;fe76acf732b2:39793 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T19:14:33,714 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T19:14:33,714 INFO [RS:1;fe76acf732b2:39793 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T19:14:33,714 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T19:14:33,714 INFO [RS:1;fe76acf732b2:39793 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T19:14:33,714 DEBUG [RS:0;fe76acf732b2:35951 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fe76acf732b2,35951,1732648471608 2024-11-26T19:14:33,714 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fe76acf732b2,35951,1732648471608' 2024-11-26T19:14:33,714 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T19:14:33,715 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T19:14:33,716 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T19:14:33,716 DEBUG [RS:0;fe76acf732b2:35951 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T19:14:33,716 INFO [RS:0;fe76acf732b2:35951 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T19:14:33,716 INFO [RS:0;fe76acf732b2:35951 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T19:14:33,716 DEBUG [RS:2;fe76acf732b2:37087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T19:14:33,716 INFO [RS:2;fe76acf732b2:37087 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T19:14:33,717 INFO [RS:2;fe76acf732b2:37087 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T19:14:33,822 INFO [RS:0;fe76acf732b2:35951 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T19:14:33,822 INFO [RS:2;fe76acf732b2:37087 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T19:14:33,826 INFO [RS:1;fe76acf732b2:39793 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T19:14:33,826 INFO [RS:2;fe76acf732b2:37087 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fe76acf732b2%2C37087%2C1732648471827, suffix=, logDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,37087,1732648471827, archiveDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/oldWALs, maxLogs=32 2024-11-26T19:14:33,828 INFO [RS:0;fe76acf732b2:35951 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fe76acf732b2%2C35951%2C1732648471608, suffix=, logDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,35951,1732648471608, archiveDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/oldWALs, maxLogs=32 2024-11-26T19:14:33,838 INFO [RS:1;fe76acf732b2:39793 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fe76acf732b2%2C39793%2C1732648471742, suffix=, logDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742, archiveDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/oldWALs, maxLogs=32 2024-11-26T19:14:33,851 DEBUG [RS:2;fe76acf732b2:37087 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,37087,1732648471827/fe76acf732b2%2C37087%2C1732648471827.1732648473830, exclude list is [], retry=0 2024-11-26T19:14:33,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38815,DS-d8f8d6e7-0acd-480c-8a0f-0732624c813b,DISK] 2024-11-26T19:14:33,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37015,DS-b118e9ce-ffe5-4017-9420-f930aa963eec,DISK] 2024-11-26T19:14:33,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39887,DS-79d5558f-903b-428b-9867-cd303d28a575,DISK] 2024-11-26T19:14:33,871 DEBUG [RS:1;fe76acf732b2:39793 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742/fe76acf732b2%2C39793%2C1732648471742.1732648473840, exclude list is [], retry=0 2024-11-26T19:14:33,877 DEBUG [RS:0;fe76acf732b2:35951 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,35951,1732648471608/fe76acf732b2%2C35951%2C1732648471608.1732648473831, exclude list is [], retry=0 2024-11-26T19:14:33,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38815,DS-d8f8d6e7-0acd-480c-8a0f-0732624c813b,DISK] 2024-11-26T19:14:33,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39887,DS-79d5558f-903b-428b-9867-cd303d28a575,DISK] 2024-11-26T19:14:33,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37015,DS-b118e9ce-ffe5-4017-9420-f930aa963eec,DISK] 2024-11-26T19:14:33,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37015,DS-b118e9ce-ffe5-4017-9420-f930aa963eec,DISK] 2024-11-26T19:14:33,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39887,DS-79d5558f-903b-428b-9867-cd303d28a575,DISK] 2024-11-26T19:14:33,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38815,DS-d8f8d6e7-0acd-480c-8a0f-0732624c813b,DISK] 2024-11-26T19:14:33,907 INFO [RS:2;fe76acf732b2:37087 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,37087,1732648471827/fe76acf732b2%2C37087%2C1732648471827.1732648473830 2024-11-26T19:14:33,931 DEBUG [RS:2;fe76acf732b2:37087 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43795:43795),(127.0.0.1/127.0.0.1:36327:36327),(127.0.0.1/127.0.0.1:33867:33867)] 2024-11-26T19:14:33,978 INFO [RS:0;fe76acf732b2:35951 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,35951,1732648471608/fe76acf732b2%2C35951%2C1732648471608.1732648473831 2024-11-26T19:14:33,985 DEBUG [RS:0;fe76acf732b2:35951 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33867:33867),(127.0.0.1/127.0.0.1:43795:43795),(127.0.0.1/127.0.0.1:36327:36327)] 2024-11-26T19:14:33,998 INFO [RS:1;fe76acf732b2:39793 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742/fe76acf732b2%2C39793%2C1732648471742.1732648473840 2024-11-26T19:14:33,999 DEBUG [RS:1;fe76acf732b2:39793 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43795:43795),(127.0.0.1/127.0.0.1:36327:36327),(127.0.0.1/127.0.0.1:33867:33867)] 2024-11-26T19:14:34,107 DEBUG [fe76acf732b2:39807 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-26T19:14:34,124 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(204): Hosts are {fe76acf732b2=0} racks are {/default-rack=0} 2024-11-26T19:14:34,132 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T19:14:34,132 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T19:14:34,132 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T19:14:34,132 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T19:14:34,132 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T19:14:34,132 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T19:14:34,133 INFO [fe76acf732b2:39807 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T19:14:34,133 INFO [fe76acf732b2:39807 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T19:14:34,133 INFO [fe76acf732b2:39807 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T19:14:34,133 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T19:14:34,142 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:34,151 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,39793,1732648471742, state=OPENING 2024-11-26T19:14:34,157 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T19:14:34,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:34,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:34,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:34,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:34,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:34,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:34,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:34,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:34,168 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:34,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,39793,1732648471742}] 2024-11-26T19:14:34,352 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T19:14:34,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41321, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T19:14:34,367 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T19:14:34,367 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T19:14:34,368 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T19:14:34,372 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fe76acf732b2%2C39793%2C1732648471742.meta, suffix=.meta, logDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742, archiveDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/oldWALs, maxLogs=32 2024-11-26T19:14:34,396 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, exclude list is [], retry=0 2024-11-26T19:14:34,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38815,DS-d8f8d6e7-0acd-480c-8a0f-0732624c813b,DISK] 2024-11-26T19:14:34,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39887,DS-79d5558f-903b-428b-9867-cd303d28a575,DISK] 2024-11-26T19:14:34,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37015,DS-b118e9ce-ffe5-4017-9420-f930aa963eec,DISK] 2024-11-26T19:14:34,430 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta 2024-11-26T19:14:34,431 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36327:36327),(127.0.0.1/127.0.0.1:33867:33867),(127.0.0.1/127.0.0.1:43795:43795)] 2024-11-26T19:14:34,431 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:14:34,434 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T19:14:34,438 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T19:14:34,445 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T19:14:34,451 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T19:14:34,452 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:34,452 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T19:14:34,452 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T19:14:34,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T19:14:34,461 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T19:14:34,461 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:34,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:34,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T19:14:34,464 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T19:14:34,465 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:34,465 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:34,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T19:14:34,467 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T19:14:34,467 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:34,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:34,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T19:14:34,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T19:14:34,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:34,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:34,476 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T19:14:34,478 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740 2024-11-26T19:14:34,482 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740 2024-11-26T19:14:34,486 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T19:14:34,487 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T19:14:34,488 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T19:14:34,495 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T19:14:34,498 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60525250, jitterRate=-0.09810349345207214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T19:14:34,498 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T19:14:34,501 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732648474453Writing region info on filesystem at 1732648474453Initializing all the Stores at 1732648474455 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648474455Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648474456 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648474456Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648474456Cleaning up temporary data from old regions at 1732648474487 (+31 ms)Running coprocessor post-open hooks at 1732648474498 (+11 ms)Region opened successfully at 1732648474501 (+3 ms) 2024-11-26T19:14:34,513 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732648474342 2024-11-26T19:14:34,526 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T19:14:34,527 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T19:14:34,530 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:34,533 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,39793,1732648471742, state=OPEN 2024-11-26T19:14:34,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:34,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:34,536 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:34,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:34,536 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:34,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:34,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:34,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:34,537 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:34,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T19:14:34,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,39793,1732648471742 in 367 msec 2024-11-26T19:14:34,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T19:14:34,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1300 sec 2024-11-26T19:14:34,564 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:34,565 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T19:14:34,593 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:34,598 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1] 2024-11-26T19:14:34,633 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:34,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49943, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:34,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6570 sec 2024-11-26T19:14:34,679 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732648474678, completionTime=-1 2024-11-26T19:14:34,681 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-26T19:14:34,682 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:34,775 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-26T19:14:34,775 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648534775 2024-11-26T19:14:34,775 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648594775 2024-11-26T19:14:34,775 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 93 msec 2024-11-26T19:14:34,777 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:34,790 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,39807,1732648470659-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:34,790 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,39807,1732648470659-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:34,800 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,39807,1732648470659-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:34,802 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fe76acf732b2:39807, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:34,802 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:34,816 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:34,824 DEBUG [master/fe76acf732b2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T19:14:34,849 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.883sec 2024-11-26T19:14:34,851 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T19:14:34,853 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T19:14:34,854 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T19:14:34,854 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T19:14:34,854 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T19:14:34,855 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,39807,1732648470659-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T19:14:34,856 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,39807,1732648470659-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T19:14:34,866 DEBUG [master/fe76acf732b2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T19:14:34,867 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T19:14:34,867 INFO [master/fe76acf732b2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fe76acf732b2,39807,1732648470659-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T19:14:34,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cf0f04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:34,907 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T19:14:34,907 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T19:14:34,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fe76acf732b2,39807,-1 for getting cluster id 2024-11-26T19:14:34,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T19:14:34,925 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4344240a-10ff-4a1f-a2a7-857ca6573a99' 2024-11-26T19:14:34,929 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T19:14:34,929 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4344240a-10ff-4a1f-a2a7-857ca6573a99" 2024-11-26T19:14:34,929 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76dcdd05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:34,929 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fe76acf732b2,39807,-1] 2024-11-26T19:14:34,933 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T19:14:34,934 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:34,936 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T19:14:34,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200d27c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T19:14:34,939 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:34,948 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1] 2024-11-26T19:14:34,948 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:34,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:34,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fe76acf732b2,39807,1732648470659 2024-11-26T19:14:34,978 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T19:14:34,984 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fe76acf732b2,39807,1732648470659 2024-11-26T19:14:34,987 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ed17eef 2024-11-26T19:14:34,988 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T19:14:34,994 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T19:14:35,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T19:14:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:35,017 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T19:14:35,020 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:35,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-26T19:14:35,024 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T19:14:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:14:35,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741837_1013 (size=358) 2024-11-26T19:14:35,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741837_1013 (size=358) 2024-11-26T19:14:35,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741837_1013 (size=358) 2024-11-26T19:14:35,070 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 02a6e477a3fdc6610cd970f26fa4f748, NAME => 'test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,070 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e21282686b8b13bb616e2402c265eba0, NAME => 'test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,077 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 604c89e3d00f984e4d556569da533f59, NAME => 'test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,093 WARN [IPC Server handler 0 on default port 45145 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T19:14:35,093 WARN [IPC Server handler 0 on default port 45145 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T19:14:35,093 WARN [IPC Server handler 0 on default port 45145 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T19:14:35,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741838_1014 (size=42) 2024-11-26T19:14:35,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741838_1014 (size=42) 2024-11-26T19:14:35,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 02a6e477a3fdc6610cd970f26fa4f748, disabling compactions & flushes 2024-11-26T19:14:35,114 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:35,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:35,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. after waiting 0 ms 2024-11-26T19:14:35,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:35,114 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:35,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 02a6e477a3fdc6610cd970f26fa4f748: Waiting for close lock at 1732648475114Disabling compacts and flushes for region at 1732648475114Disabling writes for close at 1732648475114Writing region close event to WAL at 1732648475114Closed at 1732648475114 2024-11-26T19:14:35,115 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 17b4309496c03b5e59c2108319874f01, NAME => 'test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741829_1005 (size=34) 2024-11-26T19:14:35,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741839_1015 (size=45) 2024-11-26T19:14:35,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741839_1015 (size=45) 2024-11-26T19:14:35,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741840_1016 (size=45) 2024-11-26T19:14:35,141 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741840_1016 (size=45) 2024-11-26T19:14:35,142 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e21282686b8b13bb616e2402c265eba0, disabling compactions & flushes 2024-11-26T19:14:35,142 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:35,142 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:35,142 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. after waiting 0 ms 2024-11-26T19:14:35,142 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:35,142 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:35,142 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e21282686b8b13bb616e2402c265eba0: Waiting for close lock at 1732648475142Disabling compacts and flushes for region at 1732648475142Disabling writes for close at 1732648475142Writing region close event to WAL at 1732648475142Closed at 1732648475142 2024-11-26T19:14:35,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741839_1015 (size=45) 2024-11-26T19:14:35,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741840_1016 (size=45) 2024-11-26T19:14:35,143 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 46694b0adbd3389dc584d68e34855bba, NAME => 'test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 604c89e3d00f984e4d556569da533f59, disabling compactions & flushes 2024-11-26T19:14:35,144 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:35,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:35,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. after waiting 0 ms 2024-11-26T19:14:35,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:35,144 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:35,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 604c89e3d00f984e4d556569da533f59: Waiting for close lock at 1732648475144Disabling compacts and flushes for region at 1732648475144Disabling writes for close at 1732648475144Writing region close event to WAL at 1732648475144Closed at 1732648475144 2024-11-26T19:14:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:14:35,145 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 80e1f97febf19796e5899d139b69e706, NAME => 'test,fff,1732648474996.80e1f97febf19796e5899d139b69e706.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741841_1017 (size=45) 2024-11-26T19:14:35,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741841_1017 (size=45) 2024-11-26T19:14:35,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741841_1017 (size=45) 2024-11-26T19:14:35,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 17b4309496c03b5e59c2108319874f01, disabling compactions & flushes 2024-11-26T19:14:35,163 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:35,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:35,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. after waiting 0 ms 2024-11-26T19:14:35,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:35,163 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:35,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 17b4309496c03b5e59c2108319874f01: Waiting for close lock at 1732648475163Disabling compacts and flushes for region at 1732648475163Disabling writes for close at 1732648475163Writing region close event to WAL at 1732648475163Closed at 1732648475163 2024-11-26T19:14:35,164 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 95dce235d9eac4994cdb66aed61e67be, NAME => 'test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741842_1018 (size=45) 2024-11-26T19:14:35,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741842_1018 (size=45) 2024-11-26T19:14:35,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741842_1018 (size=45) 2024-11-26T19:14:35,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 46694b0adbd3389dc584d68e34855bba, disabling compactions & flushes 2024-11-26T19:14:35,186 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:35,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:35,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. after waiting 0 ms 2024-11-26T19:14:35,186 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:35,186 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:35,187 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 46694b0adbd3389dc584d68e34855bba: Waiting for close lock at 1732648475186Disabling compacts and flushes for region at 1732648475186Disabling writes for close at 1732648475186Writing region close event to WAL at 1732648475186Closed at 1732648475186 2024-11-26T19:14:35,187 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6567f6d117ca6fa8954460c6e3a8a38b, NAME => 'test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741843_1019 (size=45) 2024-11-26T19:14:35,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741843_1019 (size=45) 2024-11-26T19:14:35,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741843_1019 (size=45) 2024-11-26T19:14:35,204 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1732648474996.80e1f97febf19796e5899d139b69e706.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,205 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 80e1f97febf19796e5899d139b69e706, disabling compactions & flushes 2024-11-26T19:14:35,205 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. 2024-11-26T19:14:35,205 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. 2024-11-26T19:14:35,205 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. after waiting 0 ms 2024-11-26T19:14:35,205 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. 2024-11-26T19:14:35,205 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. 2024-11-26T19:14:35,205 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 80e1f97febf19796e5899d139b69e706: Waiting for close lock at 1732648475205Disabling compacts and flushes for region at 1732648475205Disabling writes for close at 1732648475205Writing region close event to WAL at 1732648475205Closed at 1732648475205 2024-11-26T19:14:35,206 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0eefd38e6213218f00748a98ad16d4a8, NAME => 'test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741844_1020 (size=45) 2024-11-26T19:14:35,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741844_1020 (size=45) 2024-11-26T19:14:35,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741844_1020 (size=45) 2024-11-26T19:14:35,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 95dce235d9eac4994cdb66aed61e67be, disabling compactions & flushes 2024-11-26T19:14:35,248 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:35,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:35,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. after waiting 0 ms 2024-11-26T19:14:35,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:35,248 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:35,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 95dce235d9eac4994cdb66aed61e67be: Waiting for close lock at 1732648475248Disabling compacts and flushes for region at 1732648475248Disabling writes for close at 1732648475248Writing region close event to WAL at 1732648475248Closed at 1732648475248 2024-11-26T19:14:35,249 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a405982b7a6ac45a1ee2f1e141411d73, NAME => 'test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741845_1021 (size=45) 2024-11-26T19:14:35,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741845_1021 (size=45) 2024-11-26T19:14:35,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741845_1021 (size=45) 2024-11-26T19:14:35,274 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,274 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6567f6d117ca6fa8954460c6e3a8a38b, disabling compactions & flushes 2024-11-26T19:14:35,274 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:35,274 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:35,274 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. after waiting 0 ms 2024-11-26T19:14:35,274 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:35,274 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:35,274 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6567f6d117ca6fa8954460c6e3a8a38b: Waiting for close lock at 1732648475274Disabling compacts and flushes for region at 1732648475274Disabling writes for close at 1732648475274Writing region close event to WAL at 1732648475274Closed at 1732648475274 2024-11-26T19:14:35,275 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => da37a162f669759fa05d510d0a2a91e0, NAME => 'test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741846_1022 (size=45) 2024-11-26T19:14:35,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741846_1022 (size=45) 2024-11-26T19:14:35,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741846_1022 (size=45) 2024-11-26T19:14:35,312 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,312 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0eefd38e6213218f00748a98ad16d4a8, disabling compactions & flushes 2024-11-26T19:14:35,312 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:35,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:35,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. after waiting 0 ms 2024-11-26T19:14:35,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:35,313 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:35,314 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0eefd38e6213218f00748a98ad16d4a8: Waiting for close lock at 1732648475312Disabling compacts and flushes for region at 1732648475312Disabling writes for close at 1732648475313 (+1 ms)Writing region close event to WAL at 1732648475313Closed at 1732648475313 2024-11-26T19:14:35,315 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f90f88ceda5038692cb1f9ef0b051e2a, NAME => 'test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741847_1023 (size=45) 2024-11-26T19:14:35,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741847_1023 (size=45) 2024-11-26T19:14:35,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741847_1023 (size=45) 2024-11-26T19:14:35,338 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,338 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a405982b7a6ac45a1ee2f1e141411d73, disabling compactions & flushes 2024-11-26T19:14:35,338 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:35,338 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:35,338 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. after waiting 0 ms 2024-11-26T19:14:35,338 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:35,338 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:35,338 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a405982b7a6ac45a1ee2f1e141411d73: Waiting for close lock at 1732648475338Disabling compacts and flushes for region at 1732648475338Disabling writes for close at 1732648475338Writing region close event to WAL at 1732648475338Closed at 1732648475338 2024-11-26T19:14:35,339 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 59b39ef631c4901278640a3bc267a00c, NAME => 'test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:14:35,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741848_1024 (size=45) 2024-11-26T19:14:35,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741848_1024 (size=45) 2024-11-26T19:14:35,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741848_1024 (size=45) 2024-11-26T19:14:35,362 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,362 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing da37a162f669759fa05d510d0a2a91e0, disabling compactions & flushes 2024-11-26T19:14:35,362 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:35,362 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:35,362 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. after waiting 0 ms 2024-11-26T19:14:35,362 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:35,362 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:35,363 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for da37a162f669759fa05d510d0a2a91e0: Waiting for close lock at 1732648475362Disabling compacts and flushes for region at 1732648475362Disabling writes for close at 1732648475362Writing region close event to WAL at 1732648475362Closed at 1732648475362 2024-11-26T19:14:35,363 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => aa8886d00b7ca4fe8b1f911651334aff, NAME => 'test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741849_1025 (size=45) 2024-11-26T19:14:35,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741849_1025 (size=45) 2024-11-26T19:14:35,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741849_1025 (size=45) 2024-11-26T19:14:35,377 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,377 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f90f88ceda5038692cb1f9ef0b051e2a, disabling compactions & flushes 2024-11-26T19:14:35,377 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:35,377 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:35,378 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. after waiting 1 ms 2024-11-26T19:14:35,378 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:35,378 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:35,378 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f90f88ceda5038692cb1f9ef0b051e2a: Waiting for close lock at 1732648475377Disabling compacts and flushes for region at 1732648475377Disabling writes for close at 1732648475378 (+1 ms)Writing region close event to WAL at 1732648475378Closed at 1732648475378 2024-11-26T19:14:35,379 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6994ebee6aa3384b939e1d921bc57d6d, NAME => 'test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741850_1026 (size=45) 2024-11-26T19:14:35,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741850_1026 (size=45) 2024-11-26T19:14:35,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741850_1026 (size=45) 2024-11-26T19:14:35,395 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,395 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 59b39ef631c4901278640a3bc267a00c, disabling compactions & flushes 2024-11-26T19:14:35,395 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:35,395 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:35,395 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. after waiting 0 ms 2024-11-26T19:14:35,395 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:35,395 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:35,395 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 59b39ef631c4901278640a3bc267a00c: Waiting for close lock at 1732648475395Disabling compacts and flushes for region at 1732648475395Disabling writes for close at 1732648475395Writing region close event to WAL at 1732648475395Closed at 1732648475395 2024-11-26T19:14:35,396 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ee0ca46968a82bc066976dac0a0a6d1c, NAME => 'test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741852_1028 (size=45) 2024-11-26T19:14:35,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741852_1028 (size=45) 2024-11-26T19:14:35,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741852_1028 (size=45) 2024-11-26T19:14:35,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6994ebee6aa3384b939e1d921bc57d6d, disabling compactions & flushes 2024-11-26T19:14:35,421 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:35,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:35,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. after waiting 0 ms 2024-11-26T19:14:35,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:35,421 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:35,421 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6994ebee6aa3384b939e1d921bc57d6d: Waiting for close lock at 1732648475421Disabling compacts and flushes for region at 1732648475421Disabling writes for close at 1732648475421Writing region close event to WAL at 1732648475421Closed at 1732648475421 2024-11-26T19:14:35,422 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4a80bff91b1c78c52c7778c440d38079, NAME => 'test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741853_1029 (size=45) 2024-11-26T19:14:35,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741853_1029 (size=45) 2024-11-26T19:14:35,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741853_1029 (size=45) 2024-11-26T19:14:35,432 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,432 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ee0ca46968a82bc066976dac0a0a6d1c, disabling compactions & flushes 2024-11-26T19:14:35,432 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:35,432 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:35,432 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. after waiting 0 ms 2024-11-26T19:14:35,432 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:35,433 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:35,433 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ee0ca46968a82bc066976dac0a0a6d1c: Waiting for close lock at 1732648475432Disabling compacts and flushes for region at 1732648475432Disabling writes for close at 1732648475432Writing region close event to WAL at 1732648475433 (+1 ms)Closed at 1732648475433 2024-11-26T19:14:35,433 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d697b7105100e4e004671d5e4c7c0562, NAME => 'test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741851_1027 (size=45) 2024-11-26T19:14:35,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741851_1027 (size=45) 2024-11-26T19:14:35,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741851_1027 (size=45) 2024-11-26T19:14:35,454 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,454 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing aa8886d00b7ca4fe8b1f911651334aff, disabling compactions & flushes 2024-11-26T19:14:35,454 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:35,455 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:35,455 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. after waiting 0 ms 2024-11-26T19:14:35,455 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:35,455 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:35,455 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for aa8886d00b7ca4fe8b1f911651334aff: Waiting for close lock at 1732648475454Disabling compacts and flushes for region at 1732648475454Disabling writes for close at 1732648475455 (+1 ms)Writing region close event to WAL at 1732648475455Closed at 1732648475455 2024-11-26T19:14:35,456 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8a29b0b37829b046e26e10712540b43b, NAME => 'test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741854_1030 (size=45) 2024-11-26T19:14:35,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741854_1030 (size=45) 2024-11-26T19:14:35,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741854_1030 (size=45) 2024-11-26T19:14:35,465 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,466 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4a80bff91b1c78c52c7778c440d38079, disabling compactions & flushes 2024-11-26T19:14:35,466 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:35,466 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:35,466 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. after waiting 0 ms 2024-11-26T19:14:35,466 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:35,466 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:35,466 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4a80bff91b1c78c52c7778c440d38079: Waiting for close lock at 1732648475466Disabling compacts and flushes for region at 1732648475466Disabling writes for close at 1732648475466Writing region close event to WAL at 1732648475466Closed at 1732648475466 2024-11-26T19:14:35,466 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a4e0d60b18f7600d128edab0389cb1b5, NAME => 'test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741855_1031 (size=45) 2024-11-26T19:14:35,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741855_1031 (size=45) 2024-11-26T19:14:35,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741855_1031 (size=45) 2024-11-26T19:14:35,493 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,493 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d697b7105100e4e004671d5e4c7c0562, disabling compactions & flushes 2024-11-26T19:14:35,493 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:35,493 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:35,493 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. after waiting 0 ms 2024-11-26T19:14:35,494 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:35,494 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:35,494 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d697b7105100e4e004671d5e4c7c0562: Waiting for close lock at 1732648475493Disabling compacts and flushes for region at 1732648475493Disabling writes for close at 1732648475493Writing region close event to WAL at 1732648475494 (+1 ms)Closed at 1732648475494 2024-11-26T19:14:35,494 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 282d35bc91c009e5603b19e30181f7ce, NAME => 'test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741856_1032 (size=45) 2024-11-26T19:14:35,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741856_1032 (size=45) 2024-11-26T19:14:35,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741856_1032 (size=45) 2024-11-26T19:14:35,524 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,524 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8a29b0b37829b046e26e10712540b43b, disabling compactions & flushes 2024-11-26T19:14:35,524 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:35,524 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:35,524 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. after waiting 0 ms 2024-11-26T19:14:35,524 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:35,524 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:35,524 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8a29b0b37829b046e26e10712540b43b: Waiting for close lock at 1732648475524Disabling compacts and flushes for region at 1732648475524Disabling writes for close at 1732648475524Writing region close event to WAL at 1732648475524Closed at 1732648475524 2024-11-26T19:14:35,525 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4623e7cd0ae68d169e6e332f3459f4b3, NAME => 'test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741857_1033 (size=45) 2024-11-26T19:14:35,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741857_1033 (size=45) 2024-11-26T19:14:35,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741857_1033 (size=45) 2024-11-26T19:14:35,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741858_1034 (size=45) 2024-11-26T19:14:35,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741858_1034 (size=45) 2024-11-26T19:14:35,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741858_1034 (size=45) 2024-11-26T19:14:35,550 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,550 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 282d35bc91c009e5603b19e30181f7ce, disabling compactions & flushes 2024-11-26T19:14:35,550 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:35,550 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:35,551 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. after waiting 0 ms 2024-11-26T19:14:35,551 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:35,551 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:35,551 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 282d35bc91c009e5603b19e30181f7ce: Waiting for close lock at 1732648475550Disabling compacts and flushes for region at 1732648475550Disabling writes for close at 1732648475551 (+1 ms)Writing region close event to WAL at 1732648475551Closed at 1732648475551 2024-11-26T19:14:35,551 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 488125472b9f148364cde55a84ce1e55, NAME => 'test,www,1732648474996.488125472b9f148364cde55a84ce1e55.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741859_1035 (size=45) 2024-11-26T19:14:35,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741859_1035 (size=45) 2024-11-26T19:14:35,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741859_1035 (size=45) 2024-11-26T19:14:35,559 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4623e7cd0ae68d169e6e332f3459f4b3, disabling compactions & flushes 2024-11-26T19:14:35,560 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. 2024-11-26T19:14:35,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. 2024-11-26T19:14:35,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. after waiting 0 ms 2024-11-26T19:14:35,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. 2024-11-26T19:14:35,560 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. 2024-11-26T19:14:35,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4623e7cd0ae68d169e6e332f3459f4b3: Waiting for close lock at 1732648475559Disabling compacts and flushes for region at 1732648475559Disabling writes for close at 1732648475560 (+1 ms)Writing region close event to WAL at 1732648475560Closed at 1732648475560 2024-11-26T19:14:35,561 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 41f746ca5f963e44ed2cf4d43f5b61c0, NAME => 'test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741860_1036 (size=45) 2024-11-26T19:14:35,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741860_1036 (size=45) 2024-11-26T19:14:35,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741860_1036 (size=45) 2024-11-26T19:14:35,571 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1732648474996.488125472b9f148364cde55a84ce1e55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,571 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 488125472b9f148364cde55a84ce1e55, disabling compactions & flushes 2024-11-26T19:14:35,571 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:35,571 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:35,571 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732648474996.488125472b9f148364cde55a84ce1e55. after waiting 0 ms 2024-11-26T19:14:35,571 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:35,571 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:35,571 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 488125472b9f148364cde55a84ce1e55: Waiting for close lock at 1732648475571Disabling compacts and flushes for region at 1732648475571Disabling writes for close at 1732648475571Writing region close event to WAL at 1732648475571Closed at 1732648475571 2024-11-26T19:14:35,572 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f60803f73a60bfffe627ae76fbbfd2d6, NAME => 'test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741861_1037 (size=45) 2024-11-26T19:14:35,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741861_1037 (size=45) 2024-11-26T19:14:35,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741861_1037 (size=45) 2024-11-26T19:14:35,595 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,595 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 41f746ca5f963e44ed2cf4d43f5b61c0, disabling compactions & flushes 2024-11-26T19:14:35,595 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:35,596 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:35,596 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. after waiting 0 ms 2024-11-26T19:14:35,596 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:35,596 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:35,596 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 41f746ca5f963e44ed2cf4d43f5b61c0: Waiting for close lock at 1732648475595Disabling compacts and flushes for region at 1732648475595Disabling writes for close at 1732648475596 (+1 ms)Writing region close event to WAL at 1732648475596Closed at 1732648475596 2024-11-26T19:14:35,597 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 401b0d5d37e0f999b35af1fa8c724a27, NAME => 'test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e 2024-11-26T19:14:35,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741862_1038 (size=45) 2024-11-26T19:14:35,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741862_1038 (size=45) 2024-11-26T19:14:35,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741862_1038 (size=45) 2024-11-26T19:14:35,625 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,625 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f60803f73a60bfffe627ae76fbbfd2d6, disabling compactions & flushes 2024-11-26T19:14:35,626 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:35,626 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:35,626 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. after waiting 0 ms 2024-11-26T19:14:35,626 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:35,626 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:35,626 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f60803f73a60bfffe627ae76fbbfd2d6: Waiting for close lock at 1732648475625Disabling compacts and flushes for region at 1732648475625Disabling writes for close at 1732648475626 (+1 ms)Writing region close event to WAL at 1732648475626Closed at 1732648475626 2024-11-26T19:14:35,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741863_1039 (size=42) 2024-11-26T19:14:35,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741863_1039 (size=42) 2024-11-26T19:14:35,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741863_1039 (size=42) 2024-11-26T19:14:35,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 401b0d5d37e0f999b35af1fa8c724a27, disabling compactions & flushes 2024-11-26T19:14:35,637 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:35,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:35,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. after waiting 0 ms 2024-11-26T19:14:35,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:35,637 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:35,637 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 401b0d5d37e0f999b35af1fa8c724a27: Waiting for close lock at 1732648475637Disabling compacts and flushes for region at 1732648475637Disabling writes for close at 1732648475637Writing region close event to WAL at 1732648475637Closed at 1732648475637 2024-11-26T19:14:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:14:35,938 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:35,938 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a4e0d60b18f7600d128edab0389cb1b5, disabling compactions & flushes 2024-11-26T19:14:35,938 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:35,938 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:35,938 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. after waiting 0 ms 2024-11-26T19:14:35,938 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:35,938 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:35,938 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a4e0d60b18f7600d128edab0389cb1b5: Waiting for close lock at 1732648475938Disabling compacts and flushes for region at 1732648475938Disabling writes for close at 1732648475938Writing region close event to WAL at 1732648475938Closed at 1732648475938 2024-11-26T19:14:35,942 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T19:14:35,948 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,949 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,949 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,949 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,949 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,949 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732648474996.80e1f97febf19796e5899d139b69e706.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,949 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,949 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,950 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,950 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,950 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,950 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,950 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,950 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,951 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,951 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,951 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,951 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,951 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,951 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,951 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,951 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732648474996.488125472b9f148364cde55a84ce1e55.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,952 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,952 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,952 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:35,952 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732648475942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732648475942"}]},"ts":"1732648475942"} 2024-11-26T19:14:36,000 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-26T19:14:36,002 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T19:14:36,005 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648476002"}]},"ts":"1732648476002"} 2024-11-26T19:14:36,010 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-26T19:14:36,011 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {fe76acf732b2=0} racks are {/default-rack=0} 2024-11-26T19:14:36,033 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T19:14:36,033 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T19:14:36,033 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T19:14:36,033 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T19:14:36,033 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T19:14:36,033 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T19:14:36,033 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T19:14:36,033 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T19:14:36,033 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T19:14:36,033 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T19:14:36,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=02a6e477a3fdc6610cd970f26fa4f748, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e21282686b8b13bb616e2402c265eba0, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17b4309496c03b5e59c2108319874f01, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=46694b0adbd3389dc584d68e34855bba, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=95dce235d9eac4994cdb66aed61e67be, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6567f6d117ca6fa8954460c6e3a8a38b, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a405982b7a6ac45a1ee2f1e141411d73, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=da37a162f669759fa05d510d0a2a91e0, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=59b39ef631c4901278640a3bc267a00c, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aa8886d00b7ca4fe8b1f911651334aff, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6994ebee6aa3384b939e1d921bc57d6d, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4a80bff91b1c78c52c7778c440d38079, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a29b0b37829b046e26e10712540b43b, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a4e0d60b18f7600d128edab0389cb1b5, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=282d35bc91c009e5603b19e30181f7ce, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=488125472b9f148364cde55a84ce1e55, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=41f746ca5f963e44ed2cf4d43f5b61c0, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=401b0d5d37e0f999b35af1fa8c724a27, ASSIGN}] 2024-11-26T19:14:36,046 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=401b0d5d37e0f999b35af1fa8c724a27, ASSIGN 2024-11-26T19:14:36,047 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=401b0d5d37e0f999b35af1fa8c724a27, ASSIGN; state=OFFLINE, location=fe76acf732b2,37087,1732648471827; forceNewPlan=false, retain=false 2024-11-26T19:14:36,050 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:36,051 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN; state=OFFLINE, location=fe76acf732b2,39793,1732648471742; forceNewPlan=false, retain=false 2024-11-26T19:14:36,053 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=41f746ca5f963e44ed2cf4d43f5b61c0, ASSIGN 2024-11-26T19:14:36,056 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=41f746ca5f963e44ed2cf4d43f5b61c0, ASSIGN; state=OFFLINE, location=fe76acf732b2,35951,1732648471608; forceNewPlan=false, retain=false 2024-11-26T19:14:36,058 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=488125472b9f148364cde55a84ce1e55, ASSIGN 2024-11-26T19:14:36,061 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=488125472b9f148364cde55a84ce1e55, ASSIGN; state=OFFLINE, location=fe76acf732b2,37087,1732648471827; forceNewPlan=false, retain=false 2024-11-26T19:14:36,062 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:36,063 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN; state=OFFLINE, location=fe76acf732b2,39793,1732648471742; forceNewPlan=false, retain=false 2024-11-26T19:14:36,065 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=282d35bc91c009e5603b19e30181f7ce, ASSIGN 2024-11-26T19:14:36,066 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=282d35bc91c009e5603b19e30181f7ce, ASSIGN; state=OFFLINE, location=fe76acf732b2,35951,1732648471608; forceNewPlan=false, retain=false 2024-11-26T19:14:36,068 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a4e0d60b18f7600d128edab0389cb1b5, ASSIGN 2024-11-26T19:14:36,071 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a4e0d60b18f7600d128edab0389cb1b5, ASSIGN; state=OFFLINE, location=fe76acf732b2,35951,1732648471608; forceNewPlan=false, retain=false 2024-11-26T19:14:36,072 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8a29b0b37829b046e26e10712540b43b, ASSIGN 2024-11-26T19:14:36,074 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8a29b0b37829b046e26e10712540b43b, ASSIGN; state=OFFLINE, location=fe76acf732b2,37087,1732648471827; forceNewPlan=false, retain=false 2024-11-26T19:14:36,076 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:36,077 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN; state=OFFLINE, location=fe76acf732b2,39793,1732648471742; forceNewPlan=false, retain=false 2024-11-26T19:14:36,079 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4a80bff91b1c78c52c7778c440d38079, ASSIGN 2024-11-26T19:14:36,080 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4a80bff91b1c78c52c7778c440d38079, ASSIGN; state=OFFLINE, location=fe76acf732b2,35951,1732648471608; forceNewPlan=false, retain=false 2024-11-26T19:14:36,082 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:36,084 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN; state=OFFLINE, location=fe76acf732b2,39793,1732648471742; forceNewPlan=false, retain=false 2024-11-26T19:14:36,085 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6994ebee6aa3384b939e1d921bc57d6d, ASSIGN 2024-11-26T19:14:36,088 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6994ebee6aa3384b939e1d921bc57d6d, ASSIGN; state=OFFLINE, location=fe76acf732b2,35951,1732648471608; forceNewPlan=false, retain=false 2024-11-26T19:14:36,089 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=aa8886d00b7ca4fe8b1f911651334aff, ASSIGN 2024-11-26T19:14:36,092 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=aa8886d00b7ca4fe8b1f911651334aff, ASSIGN; state=OFFLINE, location=fe76acf732b2,37087,1732648471827; forceNewPlan=false, retain=false 2024-11-26T19:14:36,094 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=59b39ef631c4901278640a3bc267a00c, ASSIGN 2024-11-26T19:14:36,095 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=59b39ef631c4901278640a3bc267a00c, ASSIGN; state=OFFLINE, location=fe76acf732b2,37087,1732648471827; forceNewPlan=false, retain=false 2024-11-26T19:14:36,098 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:36,099 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN; state=OFFLINE, location=fe76acf732b2,39793,1732648471742; forceNewPlan=false, retain=false 2024-11-26T19:14:36,100 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=da37a162f669759fa05d510d0a2a91e0, ASSIGN 2024-11-26T19:14:36,103 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=da37a162f669759fa05d510d0a2a91e0, ASSIGN; state=OFFLINE, location=fe76acf732b2,35951,1732648471608; forceNewPlan=false, retain=false 2024-11-26T19:14:36,104 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a405982b7a6ac45a1ee2f1e141411d73, ASSIGN 2024-11-26T19:14:36,106 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a405982b7a6ac45a1ee2f1e141411d73, ASSIGN; state=OFFLINE, location=fe76acf732b2,37087,1732648471827; forceNewPlan=false, retain=false 2024-11-26T19:14:36,107 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:36,110 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN; state=OFFLINE, location=fe76acf732b2,39793,1732648471742; forceNewPlan=false, retain=false 2024-11-26T19:14:36,112 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6567f6d117ca6fa8954460c6e3a8a38b, ASSIGN 2024-11-26T19:14:36,116 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6567f6d117ca6fa8954460c6e3a8a38b, ASSIGN; state=OFFLINE, location=fe76acf732b2,35951,1732648471608; forceNewPlan=false, retain=false 2024-11-26T19:14:36,117 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=95dce235d9eac4994cdb66aed61e67be, ASSIGN 2024-11-26T19:14:36,118 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=95dce235d9eac4994cdb66aed61e67be, ASSIGN; state=OFFLINE, location=fe76acf732b2,37087,1732648471827; forceNewPlan=false, retain=false 2024-11-26T19:14:36,120 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:36,121 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN; state=OFFLINE, location=fe76acf732b2,39793,1732648471742; forceNewPlan=false, retain=false 2024-11-26T19:14:36,124 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=46694b0adbd3389dc584d68e34855bba, ASSIGN 2024-11-26T19:14:36,125 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=46694b0adbd3389dc584d68e34855bba, ASSIGN; state=OFFLINE, location=fe76acf732b2,35951,1732648471608; forceNewPlan=false, retain=false 2024-11-26T19:14:36,129 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17b4309496c03b5e59c2108319874f01, ASSIGN 2024-11-26T19:14:36,133 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=17b4309496c03b5e59c2108319874f01, ASSIGN; state=OFFLINE, location=fe76acf732b2,37087,1732648471827; forceNewPlan=false, retain=false 2024-11-26T19:14:36,134 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:36,136 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN; state=OFFLINE, location=fe76acf732b2,39793,1732648471742; forceNewPlan=false, retain=false 2024-11-26T19:14:36,138 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e21282686b8b13bb616e2402c265eba0, ASSIGN 2024-11-26T19:14:36,143 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e21282686b8b13bb616e2402c265eba0, ASSIGN; state=OFFLINE, location=fe76acf732b2,35951,1732648471608; forceNewPlan=false, retain=false 2024-11-26T19:14:36,144 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=02a6e477a3fdc6610cd970f26fa4f748, ASSIGN 2024-11-26T19:14:36,146 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=02a6e477a3fdc6610cd970f26fa4f748, ASSIGN; state=OFFLINE, location=fe76acf732b2,37087,1732648471827; forceNewPlan=false, retain=false 2024-11-26T19:14:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:14:36,202 INFO [fe76acf732b2:39807 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-26T19:14:36,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,204 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,205 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,206 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,207 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN because future has completed 2024-11-26T19:14:36,207 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,208 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=80e1f97febf19796e5899d139b69e706, regionState=OPENING, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,209 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=604c89e3d00f984e4d556569da533f59, regionState=OPENING, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60803f73a60bfffe627ae76fbbfd2d6, server=fe76acf732b2,39793,1732648471742}] 2024-11-26T19:14:36,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN because future has completed 2024-11-26T19:14:36,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,39793,1732648471742}] 2024-11-26T19:14:36,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN because future has completed 2024-11-26T19:14:36,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN because future has completed 2024-11-26T19:14:36,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN because future has completed 2024-11-26T19:14:36,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN because future has completed 2024-11-26T19:14:36,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN because future has completed 2024-11-26T19:14:36,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN because future has completed 2024-11-26T19:14:36,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,39793,1732648471742}] 2024-11-26T19:14:36,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,39793,1732648471742}] 2024-11-26T19:14:36,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,39793,1732648471742}] 2024-11-26T19:14:36,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,39793,1732648471742}] 2024-11-26T19:14:36,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,39793,1732648471742}] 2024-11-26T19:14:36,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure d697b7105100e4e004671d5e4c7c0562, server=fe76acf732b2,39793,1732648471742}] 2024-11-26T19:14:36,287 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,288 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=488125472b9f148364cde55a84ce1e55, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=8a29b0b37829b046e26e10712540b43b, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,291 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=59b39ef631c4901278640a3bc267a00c, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=401b0d5d37e0f999b35af1fa8c724a27, ASSIGN because future has completed 2024-11-26T19:14:36,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 401b0d5d37e0f999b35af1fa8c724a27, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:36,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=488125472b9f148364cde55a84ce1e55, ASSIGN because future has completed 2024-11-26T19:14:36,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8a29b0b37829b046e26e10712540b43b, ASSIGN because future has completed 2024-11-26T19:14:36,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=aa8886d00b7ca4fe8b1f911651334aff, ASSIGN because future has completed 2024-11-26T19:14:36,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=59b39ef631c4901278640a3bc267a00c, ASSIGN because future has completed 2024-11-26T19:14:36,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b39ef631c4901278640a3bc267a00c, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:36,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a405982b7a6ac45a1ee2f1e141411d73, ASSIGN because future has completed 2024-11-26T19:14:36,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a405982b7a6ac45a1ee2f1e141411d73, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:36,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa8886d00b7ca4fe8b1f911651334aff, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:36,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a29b0b37829b046e26e10712540b43b, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:36,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 488125472b9f148364cde55a84ce1e55, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:36,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=95dce235d9eac4994cdb66aed61e67be, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,340 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=17b4309496c03b5e59c2108319874f01, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,341 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,342 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=282d35bc91c009e5603b19e30181f7ce, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,345 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,345 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=4a80bff91b1c78c52c7778c440d38079, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=da37a162f669759fa05d510d0a2a91e0, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,347 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,347 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=46694b0adbd3389dc584d68e34855bba, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=e21282686b8b13bb616e2402c265eba0, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=95dce235d9eac4994cdb66aed61e67be, ASSIGN because future has completed 2024-11-26T19:14:36,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95dce235d9eac4994cdb66aed61e67be, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:36,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=17b4309496c03b5e59c2108319874f01, ASSIGN because future has completed 2024-11-26T19:14:36,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=02a6e477a3fdc6610cd970f26fa4f748, ASSIGN because future has completed 2024-11-26T19:14:36,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 02a6e477a3fdc6610cd970f26fa4f748, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:36,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=41f746ca5f963e44ed2cf4d43f5b61c0, ASSIGN because future has completed 2024-11-26T19:14:36,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=282d35bc91c009e5603b19e30181f7ce, ASSIGN because future has completed 2024-11-26T19:14:36,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a4e0d60b18f7600d128edab0389cb1b5, ASSIGN because future has completed 2024-11-26T19:14:36,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4a80bff91b1c78c52c7778c440d38079, ASSIGN because future has completed 2024-11-26T19:14:36,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6994ebee6aa3384b939e1d921bc57d6d, ASSIGN because future has completed 2024-11-26T19:14:36,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=da37a162f669759fa05d510d0a2a91e0, ASSIGN because future has completed 2024-11-26T19:14:36,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6567f6d117ca6fa8954460c6e3a8a38b, ASSIGN because future has completed 2024-11-26T19:14:36,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=46694b0adbd3389dc584d68e34855bba, ASSIGN because future has completed 2024-11-26T19:14:36,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e21282686b8b13bb616e2402c265eba0, ASSIGN because future has completed 2024-11-26T19:14:36,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure e21282686b8b13bb616e2402c265eba0, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:36,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46694b0adbd3389dc584d68e34855bba, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:36,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6567f6d117ca6fa8954460c6e3a8a38b, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:36,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure da37a162f669759fa05d510d0a2a91e0, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:36,397 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:36,398 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 0eefd38e6213218f00748a98ad16d4a8, NAME => 'test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-26T19:14:36,398 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,398 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,399 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,399 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,404 INFO [StoreOpener-0eefd38e6213218f00748a98ad16d4a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6994ebee6aa3384b939e1d921bc57d6d, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:36,406 INFO [StoreOpener-0eefd38e6213218f00748a98ad16d4a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0eefd38e6213218f00748a98ad16d4a8 columnFamilyName family 2024-11-26T19:14:36,407 DEBUG [StoreOpener-0eefd38e6213218f00748a98ad16d4a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,408 INFO [StoreOpener-0eefd38e6213218f00748a98ad16d4a8-1 {}] regionserver.HStore(327): Store=0eefd38e6213218f00748a98ad16d4a8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,408 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,409 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,410 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,411 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,411 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a80bff91b1c78c52c7778c440d38079, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:36,414 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4e0d60b18f7600d128edab0389cb1b5, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:36,420 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/0eefd38e6213218f00748a98ad16d4a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,421 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 0eefd38e6213218f00748a98ad16d4a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73257223, jitterRate=0.09161768853664398}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,422 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:36,423 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 0eefd38e6213218f00748a98ad16d4a8: Running coprocessor pre-open hook at 1732648476399Writing region info on filesystem at 1732648476399Initializing all the Stores at 1732648476401 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476401Cleaning up temporary data from old regions at 1732648476411 (+10 ms)Running coprocessor post-open hooks at 1732648476422 (+11 ms)Region opened successfully at 1732648476423 (+1 ms) 2024-11-26T19:14:36,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 282d35bc91c009e5603b19e30181f7ce, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:36,426 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8., pid=35, masterSystemTime=1732648476388 2024-11-26T19:14:36,429 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:36,429 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:36,429 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. 2024-11-26T19:14:36,430 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 4623e7cd0ae68d169e6e332f3459f4b3, NAME => 'test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-26T19:14:36,430 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,431 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,431 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,431 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,434 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=0eefd38e6213218f00748a98ad16d4a8, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,39793,1732648471742 because future has completed 2024-11-26T19:14:36,441 INFO [StoreOpener-4623e7cd0ae68d169e6e332f3459f4b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,443 INFO [StoreOpener-4623e7cd0ae68d169e6e332f3459f4b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4623e7cd0ae68d169e6e332f3459f4b3 columnFamilyName family 2024-11-26T19:14:36,443 DEBUG [StoreOpener-4623e7cd0ae68d169e6e332f3459f4b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,444 INFO [StoreOpener-4623e7cd0ae68d169e6e332f3459f4b3-1 {}] regionserver.HStore(327): Store=4623e7cd0ae68d169e6e332f3459f4b3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,445 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=13 2024-11-26T19:14:36,446 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,39793,1732648471742 in 183 msec 2024-11-26T19:14:36,447 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,447 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,448 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN in 410 msec 2024-11-26T19:14:36,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 41f746ca5f963e44ed2cf4d43f5b61c0, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:36,451 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,451 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T19:14:36,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39177, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T19:14:36,461 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/4623e7cd0ae68d169e6e332f3459f4b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,461 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 4623e7cd0ae68d169e6e332f3459f4b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66503350, jitterRate=-0.00902286171913147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,462 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:36,462 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 4623e7cd0ae68d169e6e332f3459f4b3: Running coprocessor pre-open hook at 1732648476431Writing region info on filesystem at 1732648476431Initializing all the Stores at 1732648476433 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476433Cleaning up temporary data from old regions at 1732648476448 (+15 ms)Running coprocessor post-open hooks at 1732648476462 (+14 ms)Region opened successfully at 1732648476462 2024-11-26T19:14:36,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17b4309496c03b5e59c2108319874f01, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:36,463 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:36,463 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => aa8886d00b7ca4fe8b1f911651334aff, NAME => 'test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-26T19:14:36,465 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,465 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,465 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3., pid=32, masterSystemTime=1732648476388 2024-11-26T19:14:36,465 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,465 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,468 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. 2024-11-26T19:14:36,468 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. 2024-11-26T19:14:36,469 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:36,469 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => f90f88ceda5038692cb1f9ef0b051e2a, NAME => 'test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-26T19:14:36,469 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,469 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,469 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,469 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,471 INFO [StoreOpener-aa8886d00b7ca4fe8b1f911651334aff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,473 INFO [StoreOpener-f90f88ceda5038692cb1f9ef0b051e2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,474 INFO [StoreOpener-aa8886d00b7ca4fe8b1f911651334aff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa8886d00b7ca4fe8b1f911651334aff columnFamilyName family 2024-11-26T19:14:36,474 DEBUG [StoreOpener-aa8886d00b7ca4fe8b1f911651334aff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,475 INFO [StoreOpener-aa8886d00b7ca4fe8b1f911651334aff-1 {}] regionserver.HStore(327): Store=aa8886d00b7ca4fe8b1f911651334aff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,475 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,476 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,478 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,478 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,479 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,483 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,485 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,487 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/aa8886d00b7ca4fe8b1f911651334aff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,39793,1732648471742 because future has completed 2024-11-26T19:14:36,492 INFO [StoreOpener-f90f88ceda5038692cb1f9ef0b051e2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f90f88ceda5038692cb1f9ef0b051e2a columnFamilyName family 2024-11-26T19:14:36,493 DEBUG [StoreOpener-f90f88ceda5038692cb1f9ef0b051e2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,493 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened aa8886d00b7ca4fe8b1f911651334aff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65381687, jitterRate=-0.025736942887306213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,493 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:36,493 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for aa8886d00b7ca4fe8b1f911651334aff: Running coprocessor pre-open hook at 1732648476465Writing region info on filesystem at 1732648476465Initializing all the Stores at 1732648476467 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476467Cleaning up temporary data from old regions at 1732648476479 (+12 ms)Running coprocessor post-open hooks at 1732648476493 (+14 ms)Region opened successfully at 1732648476493 2024-11-26T19:14:36,494 INFO [StoreOpener-f90f88ceda5038692cb1f9ef0b051e2a-1 {}] regionserver.HStore(327): Store=f90f88ceda5038692cb1f9ef0b051e2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,494 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,496 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,496 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff., pid=42, masterSystemTime=1732648476451 2024-11-26T19:14:36,496 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=26 2024-11-26T19:14:36,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,39793,1732648471742 in 275 msec 2024-11-26T19:14:36,498 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,498 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,499 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:36,499 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:36,499 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:36,500 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 02a6e477a3fdc6610cd970f26fa4f748, NAME => 'test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-26T19:14:36,500 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,500 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,500 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,500 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,502 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN in 462 msec 2024-11-26T19:14:36,504 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,505 INFO [StoreOpener-02a6e477a3fdc6610cd970f26fa4f748-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,507 INFO [StoreOpener-02a6e477a3fdc6610cd970f26fa4f748-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02a6e477a3fdc6610cd970f26fa4f748 columnFamilyName family 2024-11-26T19:14:36,507 DEBUG [StoreOpener-02a6e477a3fdc6610cd970f26fa4f748-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,508 INFO [StoreOpener-02a6e477a3fdc6610cd970f26fa4f748-1 {}] regionserver.HStore(327): Store=02a6e477a3fdc6610cd970f26fa4f748/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,508 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,510 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa8886d00b7ca4fe8b1f911651334aff, server=fe76acf732b2,37087,1732648471827 because future has completed 2024-11-26T19:14:36,510 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,511 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,511 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=18 2024-11-26T19:14:36,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure aa8886d00b7ca4fe8b1f911651334aff, server=fe76acf732b2,37087,1732648471827 in 194 msec 2024-11-26T19:14:36,516 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,517 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f90f88ceda5038692cb1f9ef0b051e2a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,518 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened f90f88ceda5038692cb1f9ef0b051e2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69200720, jitterRate=0.031171083450317383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,518 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:36,518 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for f90f88ceda5038692cb1f9ef0b051e2a: Running coprocessor pre-open hook at 1732648476470Writing region info on filesystem at 1732648476470Initializing all the Stores at 1732648476471 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476471Cleaning up temporary data from old regions at 1732648476498 (+27 ms)Running coprocessor post-open hooks at 1732648476518 (+20 ms)Region opened successfully at 1732648476518 2024-11-26T19:14:36,519 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a., pid=36, masterSystemTime=1732648476388 2024-11-26T19:14:36,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=aa8886d00b7ca4fe8b1f911651334aff, ASSIGN in 480 msec 2024-11-26T19:14:36,522 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:36,522 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:36,523 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:36,523 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 604c89e3d00f984e4d556569da533f59, NAME => 'test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-26T19:14:36,523 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,523 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,523 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,524 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,524 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,526 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fe76acf732b2,39793,1732648471742, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T19:14:36,529 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T19:14:36,529 INFO [StoreOpener-604c89e3d00f984e4d556569da533f59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,39793,1732648471742 because future has completed 2024-11-26T19:14:36,531 INFO [StoreOpener-604c89e3d00f984e4d556569da533f59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 604c89e3d00f984e4d556569da533f59 columnFamilyName family 2024-11-26T19:14:36,532 DEBUG [StoreOpener-604c89e3d00f984e4d556569da533f59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,532 INFO [StoreOpener-604c89e3d00f984e4d556569da533f59-1 {}] regionserver.HStore(327): Store=604c89e3d00f984e4d556569da533f59/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,533 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,533 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/02a6e477a3fdc6610cd970f26fa4f748/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,534 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,535 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 02a6e477a3fdc6610cd970f26fa4f748; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68505021, jitterRate=0.020804360508918762}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,535 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:36,535 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 02a6e477a3fdc6610cd970f26fa4f748: Running coprocessor pre-open hook at 1732648476501Writing region info on filesystem at 1732648476501Initializing all the Stores at 1732648476503 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476503Cleaning up temporary data from old regions at 1732648476511 (+8 ms)Running coprocessor post-open hooks at 1732648476535 (+24 ms)Region opened successfully at 1732648476535 2024-11-26T19:14:36,536 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,536 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748., pid=46, masterSystemTime=1732648476451 2024-11-26T19:14:36,537 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,537 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,540 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,542 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:36,542 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:36,543 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:36,543 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 59b39ef631c4901278640a3bc267a00c, NAME => 'test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-26T19:14:36,543 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,543 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,544 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,544 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=16 2024-11-26T19:14:36,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,39793,1732648471742 in 266 msec 2024-11-26T19:14:36,550 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/604c89e3d00f984e4d556569da533f59/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54729, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T19:14:36,551 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 604c89e3d00f984e4d556569da533f59; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63486384, jitterRate=-0.05397915840148926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,551 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:36,551 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 604c89e3d00f984e4d556569da533f59: Running coprocessor pre-open hook at 1732648476524Writing region info on filesystem at 1732648476524Initializing all the Stores at 1732648476526 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476526Cleaning up temporary data from old regions at 1732648476537 (+11 ms)Running coprocessor post-open hooks at 1732648476551 (+14 ms)Region opened successfully at 1732648476551 2024-11-26T19:14:36,552 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59., pid=34, masterSystemTime=1732648476388 2024-11-26T19:14:36,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN in 514 msec 2024-11-26T19:14:36,556 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,556 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:36,556 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:36,556 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:36,557 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => ee0ca46968a82bc066976dac0a0a6d1c, NAME => 'test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-26T19:14:36,557 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,557 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,557 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,558 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,561 INFO [StoreOpener-59b39ef631c4901278640a3bc267a00c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,561 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=604c89e3d00f984e4d556569da533f59, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 02a6e477a3fdc6610cd970f26fa4f748, server=fe76acf732b2,37087,1732648471827 because future has completed 2024-11-26T19:14:36,564 INFO [StoreOpener-59b39ef631c4901278640a3bc267a00c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59b39ef631c4901278640a3bc267a00c columnFamilyName family 2024-11-26T19:14:36,564 DEBUG [StoreOpener-59b39ef631c4901278640a3bc267a00c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,565 INFO [StoreOpener-59b39ef631c4901278640a3bc267a00c-1 {}] regionserver.HStore(327): Store=59b39ef631c4901278640a3bc267a00c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,565 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,39793,1732648471742 because future has completed 2024-11-26T19:14:36,566 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,567 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,568 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,568 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,571 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=5 2024-11-26T19:14:36,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 02a6e477a3fdc6610cd970f26fa4f748, server=fe76acf732b2,37087,1732648471827 in 207 msec 2024-11-26T19:14:36,575 INFO [StoreOpener-ee0ca46968a82bc066976dac0a0a6d1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=02a6e477a3fdc6610cd970f26fa4f748, ASSIGN in 538 msec 2024-11-26T19:14:36,578 INFO [StoreOpener-ee0ca46968a82bc066976dac0a0a6d1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee0ca46968a82bc066976dac0a0a6d1c columnFamilyName family 2024-11-26T19:14:36,578 DEBUG [StoreOpener-ee0ca46968a82bc066976dac0a0a6d1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,579 INFO [StoreOpener-ee0ca46968a82bc066976dac0a0a6d1c-1 {}] regionserver.HStore(327): Store=ee0ca46968a82bc066976dac0a0a6d1c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,580 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,581 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,581 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,582 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:36,582 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,582 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,584 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => da37a162f669759fa05d510d0a2a91e0, NAME => 'test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-26T19:14:36,585 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,585 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,585 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,585 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,595 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/59b39ef631c4901278640a3bc267a00c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=7 2024-11-26T19:14:36,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,39793,1732648471742 in 326 msec 2024-11-26T19:14:36,596 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,596 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 59b39ef631c4901278640a3bc267a00c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61212637, jitterRate=-0.08786062896251678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,596 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:36,596 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 59b39ef631c4901278640a3bc267a00c: Running coprocessor pre-open hook at 1732648476544Writing region info on filesystem at 1732648476544Initializing all the Stores at 1732648476545 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476545Cleaning up temporary data from old regions at 1732648476568 (+23 ms)Running coprocessor post-open hooks at 1732648476596 (+28 ms)Region opened successfully at 1732648476596 2024-11-26T19:14:36,598 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c., pid=40, masterSystemTime=1732648476451 2024-11-26T19:14:36,610 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:36,610 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:36,610 INFO [StoreOpener-da37a162f669759fa05d510d0a2a91e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,610 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/ee0ca46968a82bc066976dac0a0a6d1c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,610 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:36,610 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 488125472b9f148364cde55a84ce1e55, NAME => 'test,www,1732648474996.488125472b9f148364cde55a84ce1e55.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-26T19:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,www,1732648474996.488125472b9f148364cde55a84ce1e55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,611 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened ee0ca46968a82bc066976dac0a0a6d1c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66463862, jitterRate=-0.009611278772354126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for ee0ca46968a82bc066976dac0a0a6d1c: Running coprocessor pre-open hook at 1732648476558Writing region info on filesystem at 1732648476558Initializing all the Stores at 1732648476561 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476561Cleaning up temporary data from old regions at 1732648476582 (+21 ms)Running coprocessor post-open hooks at 1732648476611 (+29 ms)Region opened successfully at 1732648476611 2024-11-26T19:14:36,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN in 559 msec 2024-11-26T19:14:36,613 INFO [StoreOpener-da37a162f669759fa05d510d0a2a91e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da37a162f669759fa05d510d0a2a91e0 columnFamilyName family 2024-11-26T19:14:36,613 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c., pid=37, masterSystemTime=1732648476388 2024-11-26T19:14:36,613 DEBUG [StoreOpener-da37a162f669759fa05d510d0a2a91e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,614 INFO [StoreOpener-488125472b9f148364cde55a84ce1e55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,614 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,616 INFO [StoreOpener-da37a162f669759fa05d510d0a2a91e0-1 {}] regionserver.HStore(327): Store=da37a162f669759fa05d510d0a2a91e0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,616 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,618 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:36,618 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:36,618 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. 2024-11-26T19:14:36,618 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 80e1f97febf19796e5899d139b69e706, NAME => 'test,fff,1732648474996.80e1f97febf19796e5899d139b69e706.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-26T19:14:36,619 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,619 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,fff,1732648474996.80e1f97febf19796e5899d139b69e706.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,619 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59b39ef631c4901278640a3bc267a00c, server=fe76acf732b2,37087,1732648471827 because future has completed 2024-11-26T19:14:36,619 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,619 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,621 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,624 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,624 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,39793,1732648471742 because future has completed 2024-11-26T19:14:36,629 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,630 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fe76acf732b2,39793,1732648471742, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T19:14:36,633 INFO [StoreOpener-80e1f97febf19796e5899d139b69e706-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=17 2024-11-26T19:14:36,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 59b39ef631c4901278640a3bc267a00c, server=fe76acf732b2,37087,1732648471827 in 319 msec 2024-11-26T19:14:36,636 INFO [StoreOpener-80e1f97febf19796e5899d139b69e706-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80e1f97febf19796e5899d139b69e706 columnFamilyName family 2024-11-26T19:14:36,636 DEBUG [StoreOpener-80e1f97febf19796e5899d139b69e706-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,636 INFO [StoreOpener-488125472b9f148364cde55a84ce1e55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 488125472b9f148364cde55a84ce1e55 columnFamilyName family 2024-11-26T19:14:36,636 DEBUG [StoreOpener-488125472b9f148364cde55a84ce1e55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,637 INFO [StoreOpener-80e1f97febf19796e5899d139b69e706-1 {}] regionserver.HStore(327): Store=80e1f97febf19796e5899d139b69e706/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,637 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,637 INFO [StoreOpener-488125472b9f148364cde55a84ce1e55-1 {}] regionserver.HStore(327): Store=488125472b9f148364cde55a84ce1e55/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,637 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,638 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,638 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,638 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,639 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,639 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,639 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,639 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,640 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=59b39ef631c4901278640a3bc267a00c, ASSIGN in 600 msec 2024-11-26T19:14:36,642 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,642 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,646 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/da37a162f669759fa05d510d0a2a91e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,647 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened da37a162f669759fa05d510d0a2a91e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63047588, jitterRate=-0.060517728328704834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,647 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:36,648 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for da37a162f669759fa05d510d0a2a91e0: Running coprocessor pre-open hook at 1732648476585Writing region info on filesystem at 1732648476585Initializing all the Stores at 1732648476600 (+15 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476600Cleaning up temporary data from old regions at 1732648476624 (+24 ms)Running coprocessor post-open hooks at 1732648476647 (+23 ms)Region opened successfully at 1732648476647 2024-11-26T19:14:36,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=20 2024-11-26T19:14:36,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,39793,1732648471742 in 368 msec 2024-11-26T19:14:36,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN in 613 msec 2024-11-26T19:14:36,656 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0., pid=50, masterSystemTime=1732648476529 2024-11-26T19:14:36,659 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:36,659 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:36,659 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:36,659 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 6994ebee6aa3384b939e1d921bc57d6d, NAME => 'test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-26T19:14:36,660 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,660 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,660 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,660 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,662 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure da37a162f669759fa05d510d0a2a91e0, server=fe76acf732b2,35951,1732648471608 because future has completed 2024-11-26T19:14:36,670 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/488125472b9f148364cde55a84ce1e55/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,671 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 488125472b9f148364cde55a84ce1e55; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62491651, jitterRate=-0.06880183517932892}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,671 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:36,671 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 488125472b9f148364cde55a84ce1e55: Running coprocessor pre-open hook at 1732648476611Writing region info on filesystem at 1732648476611Initializing all the Stores at 1732648476613 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476613Cleaning up temporary data from old regions at 1732648476640 (+27 ms)Running coprocessor post-open hooks at 1732648476671 (+31 ms)Region opened successfully at 1732648476671 2024-11-26T19:14:36,673 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732648474996.488125472b9f148364cde55a84ce1e55., pid=44, masterSystemTime=1732648476451 2024-11-26T19:14:36,673 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/80e1f97febf19796e5899d139b69e706/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,674 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 80e1f97febf19796e5899d139b69e706; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69554949, jitterRate=0.036449506878852844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,674 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:36,675 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 80e1f97febf19796e5899d139b69e706: Running coprocessor pre-open hook at 1732648476619Writing region info on filesystem at 1732648476619Initializing all the Stores at 1732648476621 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476622 (+1 ms)Cleaning up temporary data from old regions at 1732648476639 (+17 ms)Running coprocessor post-open hooks at 1732648476674 (+35 ms)Region opened successfully at 1732648476674 2024-11-26T19:14:36,676 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:36,676 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:36,676 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732648474996.80e1f97febf19796e5899d139b69e706., pid=33, masterSystemTime=1732648476388 2024-11-26T19:14:36,676 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:36,676 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 401b0d5d37e0f999b35af1fa8c724a27, NAME => 'test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-26T19:14:36,676 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,677 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,677 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,677 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,679 INFO [StoreOpener-6994ebee6aa3384b939e1d921bc57d6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,679 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. 2024-11-26T19:14:36,679 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. 2024-11-26T19:14:36,679 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:36,679 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => d697b7105100e4e004671d5e4c7c0562, NAME => 'test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-26T19:14:36,680 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,680 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,680 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,680 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=15 2024-11-26T19:14:36,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure da37a162f669759fa05d510d0a2a91e0, server=fe76acf732b2,35951,1732648471608 in 277 msec 2024-11-26T19:14:36,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=da37a162f669759fa05d510d0a2a91e0, ASSIGN in 646 msec 2024-11-26T19:14:36,687 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=80e1f97febf19796e5899d139b69e706, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,688 INFO [StoreOpener-401b0d5d37e0f999b35af1fa8c724a27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,690 INFO [StoreOpener-6994ebee6aa3384b939e1d921bc57d6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6994ebee6aa3384b939e1d921bc57d6d columnFamilyName family 2024-11-26T19:14:36,690 DEBUG [StoreOpener-6994ebee6aa3384b939e1d921bc57d6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,691 INFO [StoreOpener-6994ebee6aa3384b939e1d921bc57d6d-1 {}] regionserver.HStore(327): Store=6994ebee6aa3384b939e1d921bc57d6d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,692 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,692 INFO [StoreOpener-d697b7105100e4e004671d5e4c7c0562-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,692 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=488125472b9f148364cde55a84ce1e55, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,39793,1732648471742 because future has completed 2024-11-26T19:14:36,694 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,695 INFO [StoreOpener-401b0d5d37e0f999b35af1fa8c724a27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 401b0d5d37e0f999b35af1fa8c724a27 columnFamilyName family 2024-11-26T19:14:36,695 DEBUG [StoreOpener-401b0d5d37e0f999b35af1fa8c724a27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,695 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,696 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,696 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 488125472b9f148364cde55a84ce1e55, server=fe76acf732b2,37087,1732648471827 because future has completed 2024-11-26T19:14:36,696 INFO [StoreOpener-401b0d5d37e0f999b35af1fa8c724a27-1 {}] regionserver.HStore(327): Store=401b0d5d37e0f999b35af1fa8c724a27/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,696 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,697 INFO [StoreOpener-d697b7105100e4e004671d5e4c7c0562-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d697b7105100e4e004671d5e4c7c0562 columnFamilyName family 2024-11-26T19:14:36,697 DEBUG [StoreOpener-d697b7105100e4e004671d5e4c7c0562-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,698 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,699 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,699 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,700 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,700 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,700 INFO [StoreOpener-d697b7105100e4e004671d5e4c7c0562-1 {}] regionserver.HStore(327): Store=d697b7105100e4e004671d5e4c7c0562/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,701 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=10 2024-11-26T19:14:36,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,39793,1732648471742 in 453 msec 2024-11-26T19:14:36,702 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,703 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,704 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN in 665 msec 2024-11-26T19:14:36,705 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,705 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,709 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=27 2024-11-26T19:14:36,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 488125472b9f148364cde55a84ce1e55, server=fe76acf732b2,37087,1732648471827 in 375 msec 2024-11-26T19:14:36,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=488125472b9f148364cde55a84ce1e55, ASSIGN in 676 msec 2024-11-26T19:14:36,717 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/6994ebee6aa3384b939e1d921bc57d6d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,718 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 6994ebee6aa3384b939e1d921bc57d6d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74720188, jitterRate=0.11341756582260132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,718 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:36,718 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 6994ebee6aa3384b939e1d921bc57d6d: Running coprocessor pre-open hook at 1732648476661Writing region info on filesystem at 1732648476661Initializing all the Stores at 1732648476663 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476663Cleaning up temporary data from old regions at 1732648476696 (+33 ms)Running coprocessor post-open hooks at 1732648476718 (+22 ms)Region opened successfully at 1732648476718 2024-11-26T19:14:36,720 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d., pid=51, masterSystemTime=1732648476529 2024-11-26T19:14:36,723 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:36,723 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:36,723 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:36,723 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 41f746ca5f963e44ed2cf4d43f5b61c0, NAME => 'test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-26T19:14:36,724 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,724 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,724 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,724 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,727 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6994ebee6aa3384b939e1d921bc57d6d, server=fe76acf732b2,35951,1732648471608 because future has completed 2024-11-26T19:14:36,732 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fe76acf732b2,35951,1732648471608, table=test, region=6994ebee6aa3384b939e1d921bc57d6d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T19:14:36,737 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/401b0d5d37e0f999b35af1fa8c724a27/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,738 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 401b0d5d37e0f999b35af1fa8c724a27; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75034778, jitterRate=0.11810532212257385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,738 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:36,739 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 401b0d5d37e0f999b35af1fa8c724a27: Running coprocessor pre-open hook at 1732648476677Writing region info on filesystem at 1732648476677Initializing all the Stores at 1732648476678 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476678Cleaning up temporary data from old regions at 1732648476700 (+22 ms)Running coprocessor post-open hooks at 1732648476739 (+39 ms)Region opened successfully at 1732648476739 2024-11-26T19:14:36,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=19 2024-11-26T19:14:36,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 6994ebee6aa3384b939e1d921bc57d6d, server=fe76acf732b2,35951,1732648471608 in 329 msec 2024-11-26T19:14:36,740 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27., pid=39, masterSystemTime=1732648476451 2024-11-26T19:14:36,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6994ebee6aa3384b939e1d921bc57d6d, ASSIGN in 704 msec 2024-11-26T19:14:36,746 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:36,746 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:36,746 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:36,747 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => a405982b7a6ac45a1ee2f1e141411d73, NAME => 'test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-26T19:14:36,747 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,747 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,747 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,748 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,749 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,751 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/d697b7105100e4e004671d5e4c7c0562/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,752 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened d697b7105100e4e004671d5e4c7c0562; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69379867, jitterRate=0.03384058177471161}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,752 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:36,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 401b0d5d37e0f999b35af1fa8c724a27, server=fe76acf732b2,37087,1732648471827 because future has completed 2024-11-26T19:14:36,753 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for d697b7105100e4e004671d5e4c7c0562: Running coprocessor pre-open hook at 1732648476680Writing region info on filesystem at 1732648476680Initializing all the Stores at 1732648476682 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476682Cleaning up temporary data from old regions at 1732648476705 (+23 ms)Running coprocessor post-open hooks at 1732648476753 (+48 ms)Region opened successfully at 1732648476753 2024-11-26T19:14:36,754 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562., pid=38, masterSystemTime=1732648476388 2024-11-26T19:14:36,755 INFO [StoreOpener-41f746ca5f963e44ed2cf4d43f5b61c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,757 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:36,757 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:36,757 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:36,757 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => f60803f73a60bfffe627ae76fbbfd2d6, NAME => 'test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-26T19:14:36,757 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,758 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,758 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,758 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,758 INFO [StoreOpener-41f746ca5f963e44ed2cf4d43f5b61c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41f746ca5f963e44ed2cf4d43f5b61c0 columnFamilyName family 2024-11-26T19:14:36,758 DEBUG [StoreOpener-41f746ca5f963e44ed2cf4d43f5b61c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,759 INFO [StoreOpener-41f746ca5f963e44ed2cf4d43f5b61c0-1 {}] regionserver.HStore(327): Store=41f746ca5f963e44ed2cf4d43f5b61c0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,759 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,760 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=30 2024-11-26T19:14:36,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 401b0d5d37e0f999b35af1fa8c724a27, server=fe76acf732b2,37087,1732648471827 in 461 msec 2024-11-26T19:14:36,762 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,763 INFO [StoreOpener-f60803f73a60bfffe627ae76fbbfd2d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,763 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,763 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,764 INFO [StoreOpener-a405982b7a6ac45a1ee2f1e141411d73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,765 INFO [StoreOpener-f60803f73a60bfffe627ae76fbbfd2d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f60803f73a60bfffe627ae76fbbfd2d6 columnFamilyName family 2024-11-26T19:14:36,766 DEBUG [StoreOpener-f60803f73a60bfffe627ae76fbbfd2d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,767 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=401b0d5d37e0f999b35af1fa8c724a27, ASSIGN in 726 msec 2024-11-26T19:14:36,770 INFO [StoreOpener-f60803f73a60bfffe627ae76fbbfd2d6-1 {}] regionserver.HStore(327): Store=f60803f73a60bfffe627ae76fbbfd2d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,770 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,771 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,772 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,772 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,772 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,775 INFO [StoreOpener-a405982b7a6ac45a1ee2f1e141411d73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a405982b7a6ac45a1ee2f1e141411d73 columnFamilyName family 2024-11-26T19:14:36,775 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=d697b7105100e4e004671d5e4c7c0562, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,775 DEBUG [StoreOpener-a405982b7a6ac45a1ee2f1e141411d73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,777 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,780 INFO [StoreOpener-a405982b7a6ac45a1ee2f1e141411d73-1 {}] regionserver.HStore(327): Store=a405982b7a6ac45a1ee2f1e141411d73/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,780 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure d697b7105100e4e004671d5e4c7c0562, server=fe76acf732b2,39793,1732648471742 because future has completed 2024-11-26T19:14:36,782 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,782 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,783 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,783 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,787 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=22 2024-11-26T19:14:36,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure d697b7105100e4e004671d5e4c7c0562, server=fe76acf732b2,39793,1732648471742 in 505 msec 2024-11-26T19:14:36,793 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/41f746ca5f963e44ed2cf4d43f5b61c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN in 753 msec 2024-11-26T19:14:36,794 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 41f746ca5f963e44ed2cf4d43f5b61c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61429764, jitterRate=-0.08462518453598022}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,794 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:36,794 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 41f746ca5f963e44ed2cf4d43f5b61c0: Running coprocessor pre-open hook at 1732648476725Writing region info on filesystem at 1732648476725Initializing all the Stores at 1732648476727 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476728 (+1 ms)Cleaning up temporary data from old regions at 1732648476763 (+35 ms)Running coprocessor post-open hooks at 1732648476794 (+31 ms)Region opened successfully at 1732648476794 2024-11-26T19:14:36,796 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f60803f73a60bfffe627ae76fbbfd2d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,797 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0., pid=55, masterSystemTime=1732648476529 2024-11-26T19:14:36,797 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened f60803f73a60bfffe627ae76fbbfd2d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71350440, jitterRate=0.06320440769195557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,797 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:36,797 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for f60803f73a60bfffe627ae76fbbfd2d6: Running coprocessor pre-open hook at 1732648476758Writing region info on filesystem at 1732648476758Initializing all the Stores at 1732648476760 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476760Cleaning up temporary data from old regions at 1732648476772 (+12 ms)Running coprocessor post-open hooks at 1732648476797 (+25 ms)Region opened successfully at 1732648476797 2024-11-26T19:14:36,799 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6., pid=31, masterSystemTime=1732648476388 2024-11-26T19:14:36,802 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:36,802 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:36,802 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:36,802 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 282d35bc91c009e5603b19e30181f7ce, NAME => 'test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-26T19:14:36,803 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,803 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,803 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,803 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,804 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,805 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:36,805 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:36,808 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,39793,1732648471742 2024-11-26T19:14:36,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 41f746ca5f963e44ed2cf4d43f5b61c0, server=fe76acf732b2,35951,1732648471608 because future has completed 2024-11-26T19:14:36,810 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/a405982b7a6ac45a1ee2f1e141411d73/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,811 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened a405982b7a6ac45a1ee2f1e141411d73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60946818, jitterRate=-0.09182164072990417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,811 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:36,811 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for a405982b7a6ac45a1ee2f1e141411d73: Running coprocessor pre-open hook at 1732648476748Writing region info on filesystem at 1732648476748Initializing all the Stores at 1732648476749 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476750 (+1 ms)Cleaning up temporary data from old regions at 1732648476783 (+33 ms)Running coprocessor post-open hooks at 1732648476811 (+28 ms)Region opened successfully at 1732648476811 2024-11-26T19:14:36,813 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73., pid=41, masterSystemTime=1732648476451 2024-11-26T19:14:36,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60803f73a60bfffe627ae76fbbfd2d6, server=fe76acf732b2,39793,1732648471742 because future has completed 2024-11-26T19:14:36,816 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:36,816 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:36,816 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:36,817 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a29b0b37829b046e26e10712540b43b, NAME => 'test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-26T19:14:36,817 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,817 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,817 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,817 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,822 INFO [StoreOpener-282d35bc91c009e5603b19e30181f7ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-26T19:14:36,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure f60803f73a60bfffe627ae76fbbfd2d6, server=fe76acf732b2,39793,1732648471742 in 608 msec 2024-11-26T19:14:36,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN in 788 msec 2024-11-26T19:14:36,830 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,830 INFO [StoreOpener-282d35bc91c009e5603b19e30181f7ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 282d35bc91c009e5603b19e30181f7ce columnFamilyName family 2024-11-26T19:14:36,831 DEBUG [StoreOpener-282d35bc91c009e5603b19e30181f7ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,834 INFO [StoreOpener-8a29b0b37829b046e26e10712540b43b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,835 INFO [StoreOpener-282d35bc91c009e5603b19e30181f7ce-1 {}] regionserver.HStore(327): Store=282d35bc91c009e5603b19e30181f7ce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a405982b7a6ac45a1ee2f1e141411d73, server=fe76acf732b2,37087,1732648471827 because future has completed 2024-11-26T19:14:36,837 INFO [StoreOpener-8a29b0b37829b046e26e10712540b43b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a29b0b37829b046e26e10712540b43b columnFamilyName family 2024-11-26T19:14:36,837 DEBUG [StoreOpener-8a29b0b37829b046e26e10712540b43b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,837 INFO [StoreOpener-8a29b0b37829b046e26e10712540b43b-1 {}] regionserver.HStore(327): Store=8a29b0b37829b046e26e10712540b43b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,838 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,838 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,839 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,839 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,840 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,841 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,841 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,841 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,841 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,841 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=28 2024-11-26T19:14:36,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 41f746ca5f963e44ed2cf4d43f5b61c0, server=fe76acf732b2,35951,1732648471608 in 387 msec 2024-11-26T19:14:36,844 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,844 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=41f746ca5f963e44ed2cf4d43f5b61c0, ASSIGN in 806 msec 2024-11-26T19:14:36,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=14 2024-11-26T19:14:36,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure a405982b7a6ac45a1ee2f1e141411d73, server=fe76acf732b2,37087,1732648471827 in 538 msec 2024-11-26T19:14:36,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a405982b7a6ac45a1ee2f1e141411d73, ASSIGN in 820 msec 2024-11-26T19:14:36,861 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/282d35bc91c009e5603b19e30181f7ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,861 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/8a29b0b37829b046e26e10712540b43b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,862 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 8a29b0b37829b046e26e10712540b43b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71554500, jitterRate=0.06624513864517212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,862 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 282d35bc91c009e5603b19e30181f7ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61328227, jitterRate=-0.08613820374011993}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,862 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:36,862 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:36,862 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 8a29b0b37829b046e26e10712540b43b: Running coprocessor pre-open hook at 1732648476818Writing region info on filesystem at 1732648476818Initializing all the Stores at 1732648476819 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476819Cleaning up temporary data from old regions at 1732648476841 (+22 ms)Running coprocessor post-open hooks at 1732648476862 (+21 ms)Region opened successfully at 1732648476862 2024-11-26T19:14:36,862 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 282d35bc91c009e5603b19e30181f7ce: Running coprocessor pre-open hook at 1732648476803Writing region info on filesystem at 1732648476803Initializing all the Stores at 1732648476805 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476805Cleaning up temporary data from old regions at 1732648476841 (+36 ms)Running coprocessor post-open hooks at 1732648476862 (+21 ms)Region opened successfully at 1732648476862 2024-11-26T19:14:36,863 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b., pid=43, masterSystemTime=1732648476451 2024-11-26T19:14:36,864 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce., pid=54, masterSystemTime=1732648476529 2024-11-26T19:14:36,868 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:36,868 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:36,868 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:36,869 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a80bff91b1c78c52c7778c440d38079, NAME => 'test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-26T19:14:36,869 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,869 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,869 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,869 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,870 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,871 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:36,871 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:36,871 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:36,871 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 95dce235d9eac4994cdb66aed61e67be, NAME => 'test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-26T19:14:36,872 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,872 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,872 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,872 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 282d35bc91c009e5603b19e30181f7ce, server=fe76acf732b2,35951,1732648471608 because future has completed 2024-11-26T19:14:36,876 INFO [StoreOpener-4a80bff91b1c78c52c7778c440d38079-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,877 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a29b0b37829b046e26e10712540b43b, server=fe76acf732b2,37087,1732648471827 because future has completed 2024-11-26T19:14:36,886 INFO [StoreOpener-4a80bff91b1c78c52c7778c440d38079-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a80bff91b1c78c52c7778c440d38079 columnFamilyName family 2024-11-26T19:14:36,886 DEBUG [StoreOpener-4a80bff91b1c78c52c7778c440d38079-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=25 2024-11-26T19:14:36,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 282d35bc91c009e5603b19e30181f7ce, server=fe76acf732b2,35951,1732648471608 in 457 msec 2024-11-26T19:14:36,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=282d35bc91c009e5603b19e30181f7ce, ASSIGN in 851 msec 2024-11-26T19:14:36,890 INFO [StoreOpener-95dce235d9eac4994cdb66aed61e67be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,893 INFO [StoreOpener-95dce235d9eac4994cdb66aed61e67be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95dce235d9eac4994cdb66aed61e67be columnFamilyName family 2024-11-26T19:14:36,893 DEBUG [StoreOpener-95dce235d9eac4994cdb66aed61e67be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,893 INFO [StoreOpener-4a80bff91b1c78c52c7778c440d38079-1 {}] regionserver.HStore(327): Store=4a80bff91b1c78c52c7778c440d38079/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,893 INFO [StoreOpener-95dce235d9eac4994cdb66aed61e67be-1 {}] regionserver.HStore(327): Store=95dce235d9eac4994cdb66aed61e67be/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,894 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,894 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,895 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=23 2024-11-26T19:14:36,895 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a29b0b37829b046e26e10712540b43b, server=fe76acf732b2,37087,1732648471827 in 566 msec 2024-11-26T19:14:36,895 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,896 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,896 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,896 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,897 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,897 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,899 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,899 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8a29b0b37829b046e26e10712540b43b, ASSIGN in 860 msec 2024-11-26T19:14:36,916 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/95dce235d9eac4994cdb66aed61e67be/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,917 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/4a80bff91b1c78c52c7778c440d38079/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,917 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 95dce235d9eac4994cdb66aed61e67be; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70469668, jitterRate=0.05007988214492798}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,917 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:36,917 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 95dce235d9eac4994cdb66aed61e67be: Running coprocessor pre-open hook at 1732648476872Writing region info on filesystem at 1732648476872Initializing all the Stores at 1732648476874 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476874Cleaning up temporary data from old regions at 1732648476896 (+22 ms)Running coprocessor post-open hooks at 1732648476917 (+21 ms)Region opened successfully at 1732648476917 2024-11-26T19:14:36,918 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 4a80bff91b1c78c52c7778c440d38079; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69561884, jitterRate=0.03655284643173218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,918 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:36,918 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 4a80bff91b1c78c52c7778c440d38079: Running coprocessor pre-open hook at 1732648476869Writing region info on filesystem at 1732648476869Initializing all the Stores at 1732648476872 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476872Cleaning up temporary data from old regions at 1732648476897 (+25 ms)Running coprocessor post-open hooks at 1732648476918 (+21 ms)Region opened successfully at 1732648476918 2024-11-26T19:14:36,919 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be., pid=45, masterSystemTime=1732648476451 2024-11-26T19:14:36,919 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079., pid=52, masterSystemTime=1732648476529 2024-11-26T19:14:36,924 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:36,924 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:36,925 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:36,925 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 17b4309496c03b5e59c2108319874f01, NAME => 'test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-26T19:14:36,925 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,925 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,926 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,926 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,929 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:36,929 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:36,929 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:36,929 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:36,929 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 46694b0adbd3389dc584d68e34855bba, NAME => 'test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-26T19:14:36,930 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,930 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,930 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,930 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,934 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95dce235d9eac4994cdb66aed61e67be, server=fe76acf732b2,37087,1732648471827 because future has completed 2024-11-26T19:14:36,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a80bff91b1c78c52c7778c440d38079, server=fe76acf732b2,35951,1732648471608 because future has completed 2024-11-26T19:14:36,943 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fe76acf732b2,35951,1732648471608, table=test, region=4a80bff91b1c78c52c7778c440d38079. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T19:14:36,944 INFO [StoreOpener-17b4309496c03b5e59c2108319874f01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,944 INFO [StoreOpener-46694b0adbd3389dc584d68e34855bba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,947 INFO [StoreOpener-17b4309496c03b5e59c2108319874f01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17b4309496c03b5e59c2108319874f01 columnFamilyName family 2024-11-26T19:14:36,947 DEBUG [StoreOpener-17b4309496c03b5e59c2108319874f01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,948 INFO [StoreOpener-17b4309496c03b5e59c2108319874f01-1 {}] regionserver.HStore(327): Store=17b4309496c03b5e59c2108319874f01/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,948 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,949 INFO [StoreOpener-46694b0adbd3389dc584d68e34855bba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46694b0adbd3389dc584d68e34855bba columnFamilyName family 2024-11-26T19:14:36,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=11 2024-11-26T19:14:36,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 95dce235d9eac4994cdb66aed61e67be, server=fe76acf732b2,37087,1732648471827 in 588 msec 2024-11-26T19:14:36,950 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,951 DEBUG [StoreOpener-46694b0adbd3389dc584d68e34855bba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:36,951 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,953 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,953 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,955 INFO [StoreOpener-46694b0adbd3389dc584d68e34855bba-1 {}] regionserver.HStore(327): Store=46694b0adbd3389dc584d68e34855bba/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:36,955 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,956 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=95dce235d9eac4994cdb66aed61e67be, ASSIGN in 914 msec 2024-11-26T19:14:36,957 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,958 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,958 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,961 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,962 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=21 2024-11-26T19:14:36,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a80bff91b1c78c52c7778c440d38079, server=fe76acf732b2,35951,1732648471608 in 549 msec 2024-11-26T19:14:36,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4a80bff91b1c78c52c7778c440d38079, ASSIGN in 930 msec 2024-11-26T19:14:36,976 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/46694b0adbd3389dc584d68e34855bba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,978 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 46694b0adbd3389dc584d68e34855bba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70938164, jitterRate=0.05706101655960083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,979 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:36,979 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 46694b0adbd3389dc584d68e34855bba: Running coprocessor pre-open hook at 1732648476930Writing region info on filesystem at 1732648476930Initializing all the Stores at 1732648476933 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476933Cleaning up temporary data from old regions at 1732648476958 (+25 ms)Running coprocessor post-open hooks at 1732648476979 (+21 ms)Region opened successfully at 1732648476979 2024-11-26T19:14:36,980 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba., pid=48, masterSystemTime=1732648476529 2024-11-26T19:14:36,984 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:36,984 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:36,985 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:36,985 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => e21282686b8b13bb616e2402c265eba0, NAME => 'test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-26T19:14:36,985 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:36,986 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:36,986 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:36,986 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:36,989 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/17b4309496c03b5e59c2108319874f01/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:36,989 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,990 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 17b4309496c03b5e59c2108319874f01; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71476019, jitterRate=0.06507568061351776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:36,990 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:36,990 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 17b4309496c03b5e59c2108319874f01: Running coprocessor pre-open hook at 1732648476926Writing region info on filesystem at 1732648476926Initializing all the Stores at 1732648476928 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476928Cleaning up temporary data from old regions at 1732648476953 (+25 ms)Running coprocessor post-open hooks at 1732648476990 (+37 ms)Region opened successfully at 1732648476990 2024-11-26T19:14:36,993 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01., pid=56, masterSystemTime=1732648476616 2024-11-26T19:14:36,997 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=48, ppid=9, state=RUNNABLE, hasLock=true; OpenRegionProcedure 46694b0adbd3389dc584d68e34855bba, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:36,998 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:36,998 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:37,002 INFO [StoreOpener-e21282686b8b13bb616e2402c265eba0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:37,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=9 2024-11-26T19:14:37,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 46694b0adbd3389dc584d68e34855bba, server=fe76acf732b2,35951,1732648471608 in 615 msec 2024-11-26T19:14:37,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=46694b0adbd3389dc584d68e34855bba, ASSIGN in 966 msec 2024-11-26T19:14:37,005 INFO [StoreOpener-e21282686b8b13bb616e2402c265eba0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e21282686b8b13bb616e2402c265eba0 columnFamilyName family 2024-11-26T19:14:37,006 DEBUG [StoreOpener-e21282686b8b13bb616e2402c265eba0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:37,008 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=17b4309496c03b5e59c2108319874f01, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:37,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17b4309496c03b5e59c2108319874f01, server=fe76acf732b2,37087,1732648471827 because future has completed 2024-11-26T19:14:37,007 INFO [StoreOpener-e21282686b8b13bb616e2402c265eba0-1 {}] regionserver.HStore(327): Store=e21282686b8b13bb616e2402c265eba0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:37,012 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:37,014 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:37,019 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:37,020 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:37,020 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:37,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=8 2024-11-26T19:14:37,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 17b4309496c03b5e59c2108319874f01, server=fe76acf732b2,37087,1732648471827 in 555 msec 2024-11-26T19:14:37,022 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:37,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=17b4309496c03b5e59c2108319874f01, ASSIGN in 986 msec 2024-11-26T19:14:37,026 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/e21282686b8b13bb616e2402c265eba0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:37,026 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened e21282686b8b13bb616e2402c265eba0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71655327, jitterRate=0.0677475780248642}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:37,026 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:37,027 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for e21282686b8b13bb616e2402c265eba0: Running coprocessor pre-open hook at 1732648476986Writing region info on filesystem at 1732648476986Initializing all the Stores at 1732648476988 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648476988Cleaning up temporary data from old regions at 1732648477020 (+32 ms)Running coprocessor post-open hooks at 1732648477026 (+6 ms)Region opened successfully at 1732648477026 2024-11-26T19:14:37,028 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0., pid=47, masterSystemTime=1732648476529 2024-11-26T19:14:37,033 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:37,033 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:37,033 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:37,033 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 6567f6d117ca6fa8954460c6e3a8a38b, NAME => 'test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-26T19:14:37,034 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,034 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:37,034 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,034 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,035 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:37,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure e21282686b8b13bb616e2402c265eba0, server=fe76acf732b2,35951,1732648471608 because future has completed 2024-11-26T19:14:37,045 INFO [StoreOpener-6567f6d117ca6fa8954460c6e3a8a38b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=6 2024-11-26T19:14:37,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure e21282686b8b13bb616e2402c265eba0, server=fe76acf732b2,35951,1732648471608 in 665 msec 2024-11-26T19:14:37,053 INFO [StoreOpener-6567f6d117ca6fa8954460c6e3a8a38b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6567f6d117ca6fa8954460c6e3a8a38b columnFamilyName family 2024-11-26T19:14:37,053 DEBUG [StoreOpener-6567f6d117ca6fa8954460c6e3a8a38b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:37,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e21282686b8b13bb616e2402c265eba0, ASSIGN in 1.0120 sec 2024-11-26T19:14:37,054 INFO [StoreOpener-6567f6d117ca6fa8954460c6e3a8a38b-1 {}] regionserver.HStore(327): Store=6567f6d117ca6fa8954460c6e3a8a38b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:37,055 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,056 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,056 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,057 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,057 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,062 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,066 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/6567f6d117ca6fa8954460c6e3a8a38b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:37,066 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 6567f6d117ca6fa8954460c6e3a8a38b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61013456, jitterRate=-0.09082865715026855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:37,067 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:37,067 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 6567f6d117ca6fa8954460c6e3a8a38b: Running coprocessor pre-open hook at 1732648477035Writing region info on filesystem at 1732648477035Initializing all the Stores at 1732648477039 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648477039Cleaning up temporary data from old regions at 1732648477057 (+18 ms)Running coprocessor post-open hooks at 1732648477067 (+10 ms)Region opened successfully at 1732648477067 2024-11-26T19:14:37,068 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b., pid=49, masterSystemTime=1732648476529 2024-11-26T19:14:37,076 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:37,077 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:37,077 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:37,077 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => a4e0d60b18f7600d128edab0389cb1b5, NAME => 'test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-26T19:14:37,077 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:37,078 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,078 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:37,078 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,078 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6567f6d117ca6fa8954460c6e3a8a38b, server=fe76acf732b2,35951,1732648471608 because future has completed 2024-11-26T19:14:37,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=12 2024-11-26T19:14:37,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 6567f6d117ca6fa8954460c6e3a8a38b, server=fe76acf732b2,35951,1732648471608 in 695 msec 2024-11-26T19:14:37,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6567f6d117ca6fa8954460c6e3a8a38b, ASSIGN in 1.0500 sec 2024-11-26T19:14:37,096 INFO [StoreOpener-a4e0d60b18f7600d128edab0389cb1b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,099 INFO [StoreOpener-a4e0d60b18f7600d128edab0389cb1b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4e0d60b18f7600d128edab0389cb1b5 columnFamilyName family 2024-11-26T19:14:37,100 DEBUG [StoreOpener-a4e0d60b18f7600d128edab0389cb1b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:37,100 INFO [StoreOpener-a4e0d60b18f7600d128edab0389cb1b5-1 {}] regionserver.HStore(327): Store=a4e0d60b18f7600d128edab0389cb1b5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:37,101 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,102 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,103 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,103 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,103 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,109 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,118 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/a4e0d60b18f7600d128edab0389cb1b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T19:14:37,120 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened a4e0d60b18f7600d128edab0389cb1b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75089241, jitterRate=0.11891688406467438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:37,120 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:37,120 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for a4e0d60b18f7600d128edab0389cb1b5: Running coprocessor pre-open hook at 1732648477078Writing region info on filesystem at 1732648477078Initializing all the Stores at 1732648477081 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648477081Cleaning up temporary data from old regions at 1732648477103 (+22 ms)Running coprocessor post-open hooks at 1732648477120 (+17 ms)Region opened successfully at 1732648477120 2024-11-26T19:14:37,122 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5., pid=53, masterSystemTime=1732648476529 2024-11-26T19:14:37,125 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:37,126 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:37,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, openSeqNum=2, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:37,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4e0d60b18f7600d128edab0389cb1b5, server=fe76acf732b2,35951,1732648471608 because future has completed 2024-11-26T19:14:37,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=24 2024-11-26T19:14:37,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure a4e0d60b18f7600d128edab0389cb1b5, server=fe76acf732b2,35951,1732648471608 in 719 msec 2024-11-26T19:14:37,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=4 2024-11-26T19:14:37,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a4e0d60b18f7600d128edab0389cb1b5, ASSIGN in 1.1060 sec 2024-11-26T19:14:37,151 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T19:14:37,151 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732648477151"}]},"ts":"1732648477151"} 2024-11-26T19:14:37,154 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-26T19:14:37,156 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T19:14:37,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 2.1500 sec 2024-11-26T19:14:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T19:14:37,186 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-26T19:14:37,186 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-26T19:14:37,188 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T19:14:37,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-26T19:14:37,213 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T19:14:37,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-26T19:14:37,214 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T19:14:37,222 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-26T19:14:37,248 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-26T19:14:37,276 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test', row='', locateType=CURRENT is [region=test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748., hostname=fe76acf732b2,37087,1732648471827, seqNum=2] 2024-11-26T19:14:37,278 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:37,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:37,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:37,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e21282686b8b13bb616e2402c265eba0 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 46694b0adbd3389dc584d68e34855bba changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6567f6d117ca6fa8954460c6e3a8a38b changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region da37a162f669759fa05d510d0a2a91e0 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6994ebee6aa3384b939e1d921bc57d6d changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4a80bff91b1c78c52c7778c440d38079 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a4e0d60b18f7600d128edab0389cb1b5 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 282d35bc91c009e5603b19e30181f7ce changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 41f746ca5f963e44ed2cf4d43f5b61c0 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 02a6e477a3fdc6610cd970f26fa4f748 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 17b4309496c03b5e59c2108319874f01 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 95dce235d9eac4994cdb66aed61e67be changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a405982b7a6ac45a1ee2f1e141411d73 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 59b39ef631c4901278640a3bc267a00c changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region aa8886d00b7ca4fe8b1f911651334aff changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8a29b0b37829b046e26e10712540b43b changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 488125472b9f148364cde55a84ce1e55 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 401b0d5d37e0f999b35af1fa8c724a27 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 604c89e3d00f984e4d556569da533f59 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 80e1f97febf19796e5899d139b69e706 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0eefd38e6213218f00748a98ad16d4a8 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f90f88ceda5038692cb1f9ef0b051e2a changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ee0ca46968a82bc066976dac0a0a6d1c changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d697b7105100e4e004671d5e4c7c0562 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4623e7cd0ae68d169e6e332f3459f4b3 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f60803f73a60bfffe627ae76fbbfd2d6 changed from -1.0 to 0.0, refreshing cache 2024-11-26T19:14:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-26T19:14:37,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-26T19:14:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-26T19:14:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(204): Hosts are {fe76acf732b2=0} racks are {/default-rack=0} 2024-11-26T19:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-26T19:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-26T19:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-26T19:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T19:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T19:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T19:14:37,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T19:14:37,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T19:14:37,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T19:14:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-26T19:14:37,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-26T19:14:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-26T19:14:37,414 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T19:14:37,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-26T19:14:37,466 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=379, OpenFileDescriptor=638, MaxFileDescriptor=1048576, SystemLoadAverage=481, ProcessCount=11, AvailableMemoryMB=6358 2024-11-26T19:14:37,473 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-26T19:14:37,474 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-26T19:14:37,477 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing fe76acf732b2,39793,1732648471742 2024-11-26T19:14:37,478 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server fe76acf732b2,39793,1732648471742: Simulated kill ***** 2024-11-26T19:14:37,478 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-26T19:14:37,482 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-26T19:14:37,486 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-26T19:14:37,487 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-26T19:14:37,487 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-26T19:14:37,487 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 313772704 }, "NonHeapMemoryUsage": { "committed": 151322624, "init": 7667712, "max": -1, "used": 149134496 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "fe76acf732b2" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "fe76acf732b2" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "fe76acf732b2" } ] } 2024-11-26T19:14:37,494 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39807 {}] master.MasterRpcServices(700): fe76acf732b2,39793,1732648471742 reported a fatal error: ***** ABORTING region server fe76acf732b2,39793,1732648471742: Simulated kill ***** 2024-11-26T19:14:37,502 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fe76acf732b2,39793,1732648471742' ***** 2024-11-26T19:14:37,502 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-26T19:14:37,502 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T19:14:37,503 INFO [RS:1;fe76acf732b2:39793 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-26T19:14:37,503 INFO [RS:1;fe76acf732b2:39793 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-26T19:14:37,503 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T19:14:37,503 INFO [RS:1;fe76acf732b2:39793 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T19:14:37,504 INFO [RS:1;fe76acf732b2:39793 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fe76acf732b2:39793. 2024-11-26T19:14:37,504 DEBUG [RS:1;fe76acf732b2:39793 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:14:37,508 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T19:14:37,511 DEBUG [RS:1;fe76acf732b2:39793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:37,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741836_1012 (size=32114) 2024-11-26T19:14:37,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741836_1012 (size=32114) 2024-11-26T19:14:37,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741836_1012 (size=32114) 2024-11-26T19:14:37,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741835_1011 (size=2237) 2024-11-26T19:14:37,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741835_1011 (size=2237) 2024-11-26T19:14:37,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741835_1011 (size=2237) 2024-11-26T19:14:37,546 DEBUG [RS:1;fe76acf732b2:39793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:37,546 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T19:14:37,547 INFO [RS:1;fe76acf732b2:39793 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39793 2024-11-26T19:14:37,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fe76acf732b2,39793,1732648471742 2024-11-26T19:14:37,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T19:14:37,558 INFO [RS:1;fe76acf732b2:39793 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T19:14:37,559 INFO [regionserver/fe76acf732b2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T19:14:37,561 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fe76acf732b2,39793,1732648471742] 2024-11-26T19:14:37,563 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fe76acf732b2,39793,1732648471742 already deleted, retry=false 2024-11-26T19:14:37,564 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of fe76acf732b2,39793,1732648471742 on fe76acf732b2,39807,1732648470659 2024-11-26T19:14:37,574 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:37,579 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for fe76acf732b2,39793,1732648471742 (carryingMeta=true) fe76acf732b2,39793,1732648471742/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2571e3e1[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T19:14:37,583 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:37,584 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true, isMeta: true 2024-11-26T19:14:37,588 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting 2024-11-26T19:14:37,590 INFO [PEWorker-1 {}] master.SplitWALManager(105): fe76acf732b2,39793,1732648471742 WAL count=1, meta=true 2024-11-26T19:14:37,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta}] 2024-11-26T19:14:37,605 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T19:14:37,605 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:37,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:37,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:14:37,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39793-0x10155b36f3b0002, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:14:37,665 INFO [RS:1;fe76acf732b2:39793 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T19:14:37,665 INFO [RS:1;fe76acf732b2:39793 {}] regionserver.HRegionServer(1031): Exiting; stopping=fe76acf732b2,39793,1732648471742; zookeeper connection closed. 2024-11-26T19:14:37,666 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@9f0b3b1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@9f0b3b1 2024-11-26T19:14:37,673 INFO [regionserver/fe76acf732b2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T19:14:37,673 INFO [regionserver/fe76acf732b2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T19:14:37,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35951 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-26T19:14:37,817 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, size=31.4 K (32114bytes) 2024-11-26T19:14:37,818 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-26T19:14:37,818 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-26T19:14:37,818 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta 2024-11-26T19:14:37,825 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta after 5ms 2024-11-26T19:14:37,836 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta took 21ms 2024-11-26T19:14:37,847 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-26T19:14:37,855 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta so closing down 2024-11-26T19:14:37,855 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T19:14:37,858 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta.temp 2024-11-26T19:14:37,865 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta.temp 2024-11-26T19:14:37,868 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T19:14:37,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741864_1040 (size=31873) 2024-11-26T19:14:37,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741864_1040 (size=31873) 2024-11-26T19:14:37,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741864_1040 (size=31873) 2024-11-26T19:14:37,884 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-26T19:14:37,889 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta.temp to hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T19:14:37,903 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 61 ms; skipped=1; WAL=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-26T19:14:37,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, journal: Splitting hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, size=31.4 K (32114bytes) at 1732648477817Finishing writing output for hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta so closing down at 1732648477855 (+38 ms)Creating recovered edits writer path=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta.temp at 1732648477865 (+10 ms)3 split writer threads finished at 1732648477868 (+3 ms)Closed recovered edits writer path=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1732648477884 (+16 ms)Rename recovered edits hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta.temp to hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1732648477889 (+5 ms)Processed 58 edits across 1 Regions in 61 ms; skipped=1; WAL=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1732648477903 (+14 ms) 2024-11-26T19:14:37,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta 2024-11-26T19:14:37,905 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-26T19:14:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-26T19:14:37,917 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta to hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/oldWALs 2024-11-26T19:14:37,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-26T19:14:37,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 in 307 msec 2024-11-26T19:14:37,925 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:37,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-26T19:14:37,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 in 334 msec 2024-11-26T19:14:37,931 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting dir is empty, no logs to split. 2024-11-26T19:14:37,931 INFO [PEWorker-1 {}] master.SplitWALManager(105): fe76acf732b2,39793,1732648471742 WAL count=0, meta=true 2024-11-26T19:14:37,931 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fe76acf732b2,39793,1732648471742 WAL splitting is done? wals=0, meta=true 2024-11-26T19:14:37,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T19:14:37,934 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:37,936 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T19:14:38,086 DEBUG [fe76acf732b2:39807 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T19:14:38,086 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(204): Hosts are {fe76acf732b2=0} racks are {/default-rack=0} 2024-11-26T19:14:38,087 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T19:14:38,087 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T19:14:38,087 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T19:14:38,087 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T19:14:38,087 INFO [fe76acf732b2:39807 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T19:14:38,087 INFO [fe76acf732b2:39807 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T19:14:38,087 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T19:14:38,087 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:38,090 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPENING 2024-11-26T19:14:38,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:38,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:38,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:38,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:38,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:38,098 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:38,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:38,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:38,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741838_1014 (size=42) 2024-11-26T19:14:38,261 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T19:14:38,261 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T19:14:38,262 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T19:14:38,264 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fe76acf732b2%2C35951%2C1732648471608.meta, suffix=.meta, logDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,35951,1732648471608, archiveDir=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/oldWALs, maxLogs=32 2024-11-26T19:14:38,284 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,35951,1732648471608/fe76acf732b2%2C35951%2C1732648471608.meta.1732648478265.meta, exclude list is [], retry=0 2024-11-26T19:14:38,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38815,DS-d8f8d6e7-0acd-480c-8a0f-0732624c813b,DISK] 2024-11-26T19:14:38,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37015,DS-b118e9ce-ffe5-4017-9420-f930aa963eec,DISK] 2024-11-26T19:14:38,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39887,DS-79d5558f-903b-428b-9867-cd303d28a575,DISK] 2024-11-26T19:14:38,302 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,35951,1732648471608/fe76acf732b2%2C35951%2C1732648471608.meta.1732648478265.meta 2024-11-26T19:14:38,303 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36327:36327),(127.0.0.1/127.0.0.1:43795:43795),(127.0.0.1/127.0.0.1:33867:33867)] 2024-11-26T19:14:38,303 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T19:14:38,304 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T19:14:38,304 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T19:14:38,304 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T19:14:38,304 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T19:14:38,304 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:38,304 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T19:14:38,304 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T19:14:38,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T19:14:38,308 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T19:14:38,308 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:38,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:38,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T19:14:38,314 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T19:14:38,314 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:38,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:38,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T19:14:38,316 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T19:14:38,316 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:38,317 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:38,317 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T19:14:38,318 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T19:14:38,318 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:38,318 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T19:14:38,319 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T19:14:38,320 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740 2024-11-26T19:14:38,323 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740 2024-11-26T19:14:38,324 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T19:14:38,346 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T19:14:38,351 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-26T19:14:38,437 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/info/d9cb4e0661924184aa927e9f96efda7c is 138, key is test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0./info:regioninfo/1732648477035/Put/seqid=0 2024-11-26T19:14:38,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741866_1042 (size=34924) 2024-11-26T19:14:38,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741866_1042 (size=34924) 2024-11-26T19:14:38,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741866_1042 (size=34924) 2024-11-26T19:14:38,475 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/info/d9cb4e0661924184aa927e9f96efda7c 2024-11-26T19:14:38,585 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/ns/db27f90462c140f3b5b8a18f4fc5bcfe is 43, key is default/ns:d/1732648474647/Put/seqid=0 2024-11-26T19:14:38,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741867_1043 (size=5153) 2024-11-26T19:14:38,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741867_1043 (size=5153) 2024-11-26T19:14:38,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741867_1043 (size=5153) 2024-11-26T19:14:38,605 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/ns/db27f90462c140f3b5b8a18f4fc5bcfe 2024-11-26T19:14:38,646 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/table/31a750f6f5e74f96855d9a9f7dbf6320 is 40, key is test/table:state/1732648477151/Put/seqid=0 2024-11-26T19:14:38,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741868_1044 (size=5165) 2024-11-26T19:14:38,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741868_1044 (size=5165) 2024-11-26T19:14:38,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741868_1044 (size=5165) 2024-11-26T19:14:38,673 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/table/31a750f6f5e74f96855d9a9f7dbf6320 2024-11-26T19:14:38,685 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/info/d9cb4e0661924184aa927e9f96efda7c as hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/info/d9cb4e0661924184aa927e9f96efda7c 2024-11-26T19:14:38,704 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/info/d9cb4e0661924184aa927e9f96efda7c, entries=260, sequenceid=60, filesize=34.1 K 2024-11-26T19:14:38,707 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/ns/db27f90462c140f3b5b8a18f4fc5bcfe as hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/ns/db27f90462c140f3b5b8a18f4fc5bcfe 2024-11-26T19:14:38,721 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/ns/db27f90462c140f3b5b8a18f4fc5bcfe, entries=2, sequenceid=60, filesize=5.0 K 2024-11-26T19:14:38,723 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/table/31a750f6f5e74f96855d9a9f7dbf6320 as hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/table/31a750f6f5e74f96855d9a9f7dbf6320 2024-11-26T19:14:38,735 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/table/31a750f6f5e74f96855d9a9f7dbf6320, entries=2, sequenceid=60, filesize=5.0 K 2024-11-26T19:14:38,742 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 389ms, sequenceid=60, compaction requested=false; wal=null 2024-11-26T19:14:38,744 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T19:14:38,748 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T19:14:38,750 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T19:14:38,750 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T19:14:38,752 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T19:14:38,754 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T19:14:38,759 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-26T19:14:38,762 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75479006, jitterRate=0.12472483515739441}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T19:14:38,763 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T19:14:38,768 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732648478305Writing region info on filesystem at 1732648478305Initializing all the Stores at 1732648478306 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648478306Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648478306Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648478306Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732648478306Obtaining lock to block concurrent updates at 1732648478351 (+45 ms)Preparing flush snapshotting stores in 1588230740 at 1732648478351Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732648478355 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732648478355Flushing 1588230740/info: creating writer at 1732648478357 (+2 ms)Flushing 1588230740/info: appending metadata at 1732648478423 (+66 ms)Flushing 1588230740/info: closing flushed file at 1732648478428 (+5 ms)Flushing 1588230740/ns: creating writer at 1732648478543 (+115 ms)Flushing 1588230740/ns: appending metadata at 1732648478579 (+36 ms)Flushing 1588230740/ns: closing flushed file at 1732648478579Flushing 1588230740/table: creating writer at 1732648478614 (+35 ms)Flushing 1588230740/table: appending metadata at 1732648478639 (+25 ms)Flushing 1588230740/table: closing flushed file at 1732648478639Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d73f9b2: reopening flushed file at 1732648478683 (+44 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2edf32f5: reopening flushed file at 1732648478706 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@637a3d22: reopening flushed file at 1732648478722 (+16 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 389ms, sequenceid=60, compaction requested=false; wal=null at 1732648478742 (+20 ms)Cleaning up temporary data from old regions at 1732648478750 (+8 ms)Running coprocessor post-open hooks at 1732648478763 (+13 ms)Region opened successfully at 1732648478763 2024-11-26T19:14:38,771 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1732648478255 2024-11-26T19:14:38,775 DEBUG [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T19:14:38,775 INFO [RS_OPEN_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T19:14:38,776 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:38,778 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:38,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:38,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:38,780 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:38,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:38,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:38,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:38,782 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:38,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-26T19:14:38,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 in 684 msec 2024-11-26T19:14:38,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-26T19:14:38,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 855 msec 2024-11-26T19:14:38,796 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): fe76acf732b2,39793,1732648471742 had 9 regions 2024-11-26T19:14:38,798 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true, isMeta: false 2024-11-26T19:14:38,802 INFO [PEWorker-1 {}] master.SplitWALManager(105): fe76acf732b2,39793,1732648471742 WAL count=1, meta=false 2024-11-26T19:14:38,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840}] 2024-11-26T19:14:38,805 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:38,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:38,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35951 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-26T19:14:38,992 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840, size=2.2 K (2237bytes) 2024-11-26T19:14:38,992 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840 2024-11-26T19:14:38,992 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840 after 0ms 2024-11-26T19:14:38,996 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840 took 5ms 2024-11-26T19:14:38,999 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 0eefd38e6213218f00748a98ad16d4a8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T19:14:39,001 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4623e7cd0ae68d169e6e332f3459f4b3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T19:14:39,003 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for f90f88ceda5038692cb1f9ef0b051e2a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T19:14:39,007 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 604c89e3d00f984e4d556569da533f59: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T19:14:39,009 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ee0ca46968a82bc066976dac0a0a6d1c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T19:14:39,012 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 80e1f97febf19796e5899d139b69e706: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T19:14:39,018 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d697b7105100e4e004671d5e4c7c0562: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T19:14:39,020 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for f60803f73a60bfffe627ae76fbbfd2d6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T19:14:39,020 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840 so closing down 2024-11-26T19:14:39,020 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T19:14:39,021 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T19:14:39,021 INFO [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 25 ms; skipped=8; WAL=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-26T19:14:39,021 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840, journal: Splitting hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840, size=2.2 K (2237bytes) at 1732648478992Finishing writing output for hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840 so closing down at 1732648479020 (+28 ms)3 split writer threads finished at 1732648479021 (+1 ms)Processed 8 edits across 0 Regions in 25 ms; skipped=8; WAL=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1732648479021 2024-11-26T19:14:39,021 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840 2024-11-26T19:14:39,021 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fe76acf732b2:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-26T19:14:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-26T19:14:39,040 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting/fe76acf732b2%2C39793%2C1732648471742.1732648473840 to hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/oldWALs 2024-11-26T19:14:39,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T19:14:39,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 in 232 msec 2024-11-26T19:14:39,047 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-26T19:14:39,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 in 245 msec 2024-11-26T19:14:39,053 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/WALs/fe76acf732b2,39793,1732648471742-splitting dir is empty, no logs to split. 2024-11-26T19:14:39,053 INFO [PEWorker-1 {}] master.SplitWALManager(105): fe76acf732b2,39793,1732648471742 WAL count=0, meta=false 2024-11-26T19:14:39,053 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fe76acf732b2,39793,1732648471742 WAL splitting is done? wals=0, meta=false 2024-11-26T19:14:39,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN}] 2024-11-26T19:14:39,060 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,062 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T19:14:39,063 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,064 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T19:14:39,067 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,071 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T19:14:39,073 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,074 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T19:14:39,076 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,079 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T19:14:39,083 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,084 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T19:14:39,085 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,090 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T19:14:39,091 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,092 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T19:14:39,212 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(204): Hosts are {fe76acf732b2=0} racks are {/default-rack=0} 2024-11-26T19:14:39,213 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T19:14:39,213 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T19:14:39,213 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T19:14:39,213 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T19:14:39,213 INFO [fe76acf732b2:39807 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T19:14:39,213 INFO [fe76acf732b2:39807 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T19:14:39,213 DEBUG [fe76acf732b2:39807 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T19:14:39,214 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,215 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,215 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,216 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=80e1f97febf19796e5899d139b69e706, regionState=OPENING, regionLocation=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,216 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fe76acf732b2:39793 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:39,217 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,217 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,218 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1, error=java.net.ConnectException: Call to address=fe76acf732b2:39793 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 2024-11-26T19:14:39,218 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=604c89e3d00f984e4d556569da533f59, regionState=OPENING, regionLocation=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-26T19:14:39,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1 from cache 2024-11-26T19:14:39,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=fe76acf732b2:39793 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 2024-11-26T19:14:39,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fe76acf732b2:39793 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 2024-11-26T19:14:39,221 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fe76acf732b2:39793 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:39,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=fe76acf732b2:39793 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 2024-11-26T19:14:39,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=fe76acf732b2:39793 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 2024-11-26T19:14:39,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=fe76acf732b2:39793 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 2024-11-26T19:14:39,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=fe76acf732b2:39793 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 2024-11-26T19:14:39,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=fe76acf732b2:39793 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 2024-11-26T19:14:39,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,39793,1732648471742, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=fe76acf732b2:39793 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 2024-11-26T19:14:39,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fe76acf732b2:39793 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fe76acf732b2/172.17.0.2:39793 2024-11-26T19:14:39,323 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T19:14:39,323 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fe76acf732b2,35951,1732648471608, seqNum=-1] 2024-11-26T19:14:39,324 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T19:14:39,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41871, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T19:14:39,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN because future has completed 2024-11-26T19:14:39,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:39,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN because future has completed 2024-11-26T19:14:39,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN because future has completed 2024-11-26T19:14:39,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN because future has completed 2024-11-26T19:14:39,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:39,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN because future has completed 2024-11-26T19:14:39,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN because future has completed 2024-11-26T19:14:39,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN because future has completed 2024-11-26T19:14:39,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:39,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN because future has completed 2024-11-26T19:14:39,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:39,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:39,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,37087,1732648471827}] 2024-11-26T19:14:39,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d697b7105100e4e004671d5e4c7c0562, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:39,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60803f73a60bfffe627ae76fbbfd2d6, server=fe76acf732b2,35951,1732648471608}] 2024-11-26T19:14:39,495 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:39,495 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => f90f88ceda5038692cb1f9ef0b051e2a, NAME => 'test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-26T19:14:39,496 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,496 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:39,496 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,496 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,511 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:39,512 INFO [StoreOpener-f90f88ceda5038692cb1f9ef0b051e2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,512 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 0eefd38e6213218f00748a98ad16d4a8, NAME => 'test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-26T19:14:39,512 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,512 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:39,512 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,513 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,513 INFO [StoreOpener-f90f88ceda5038692cb1f9ef0b051e2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f90f88ceda5038692cb1f9ef0b051e2a columnFamilyName family 2024-11-26T19:14:39,513 DEBUG [StoreOpener-f90f88ceda5038692cb1f9ef0b051e2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:39,514 INFO [StoreOpener-f90f88ceda5038692cb1f9ef0b051e2a-1 {}] regionserver.HStore(327): Store=f90f88ceda5038692cb1f9ef0b051e2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:39,514 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,515 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,517 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,517 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,517 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,520 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,520 INFO [StoreOpener-0eefd38e6213218f00748a98ad16d4a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,521 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened f90f88ceda5038692cb1f9ef0b051e2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67193671, jitterRate=0.0012637227773666382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:39,522 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:39,522 INFO [StoreOpener-0eefd38e6213218f00748a98ad16d4a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0eefd38e6213218f00748a98ad16d4a8 columnFamilyName family 2024-11-26T19:14:39,522 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for f90f88ceda5038692cb1f9ef0b051e2a: Running coprocessor pre-open hook at 1732648479500Writing region info on filesystem at 1732648479500Initializing all the Stores at 1732648479501 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648479502 (+1 ms)Cleaning up temporary data from old regions at 1732648479517 (+15 ms)Running coprocessor post-open hooks at 1732648479522 (+5 ms)Region opened successfully at 1732648479522 2024-11-26T19:14:39,522 DEBUG [StoreOpener-0eefd38e6213218f00748a98ad16d4a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:39,523 INFO [StoreOpener-0eefd38e6213218f00748a98ad16d4a8-1 {}] regionserver.HStore(327): Store=0eefd38e6213218f00748a98ad16d4a8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:39,523 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a., pid=73, masterSystemTime=1732648479489 2024-11-26T19:14:39,523 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,524 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,527 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,528 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,528 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,531 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,531 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:39,531 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:39,532 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:39,532 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => d697b7105100e4e004671d5e4c7c0562, NAME => 'test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-26T19:14:39,532 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,532 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:39,532 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,532 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 0eefd38e6213218f00748a98ad16d4a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74540635, jitterRate=0.11074201762676239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:39,532 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,532 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:39,533 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 0eefd38e6213218f00748a98ad16d4a8: Running coprocessor pre-open hook at 1732648479513Writing region info on filesystem at 1732648479513Initializing all the Stores at 1732648479515 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648479515Cleaning up temporary data from old regions at 1732648479528 (+13 ms)Running coprocessor post-open hooks at 1732648479533 (+5 ms)Region opened successfully at 1732648479533 2024-11-26T19:14:39,535 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-26T19:14:39,536 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,536 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,542 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8., pid=74, masterSystemTime=1732648479506 2024-11-26T19:14:39,545 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:39,545 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:39,545 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:39,545 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 604c89e3d00f984e4d556569da533f59, NAME => 'test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-26T19:14:39,546 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,546 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:39,546 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,546 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,554 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-26T19:14:39,556 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,556 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60803f73a60bfffe627ae76fbbfd2d6, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,557 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60803f73a60bfffe627ae76fbbfd2d6, server=fe76acf732b2,35951,1732648471608 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,561 INFO [StoreOpener-604c89e3d00f984e4d556569da533f59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,561 INFO [StoreOpener-d697b7105100e4e004671d5e4c7c0562-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,563 INFO [StoreOpener-604c89e3d00f984e4d556569da533f59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 604c89e3d00f984e4d556569da533f59 columnFamilyName family 2024-11-26T19:14:39,563 INFO [StoreOpener-d697b7105100e4e004671d5e4c7c0562-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d697b7105100e4e004671d5e4c7c0562 columnFamilyName family 2024-11-26T19:14:39,563 DEBUG [StoreOpener-d697b7105100e4e004671d5e4c7c0562-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:39,563 INFO [StoreOpener-d697b7105100e4e004671d5e4c7c0562-1 {}] regionserver.HStore(327): Store=d697b7105100e4e004671d5e4c7c0562/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:39,564 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,565 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,565 DEBUG [StoreOpener-604c89e3d00f984e4d556569da533f59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:39,567 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,567 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,567 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,570 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,570 INFO [StoreOpener-604c89e3d00f984e4d556569da533f59-1 {}] regionserver.HStore(327): Store=604c89e3d00f984e4d556569da533f59/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:39,571 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,572 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,572 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened d697b7105100e4e004671d5e4c7c0562; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66081905, jitterRate=-0.015302881598472595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:39,572 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:39,572 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for d697b7105100e4e004671d5e4c7c0562: Running coprocessor pre-open hook at 1732648479533Writing region info on filesystem at 1732648479533Initializing all the Stores at 1732648479542 (+9 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648479542Cleaning up temporary data from old regions at 1732648479567 (+25 ms)Running coprocessor post-open hooks at 1732648479572 (+5 ms)Region opened successfully at 1732648479572 2024-11-26T19:14:39,574 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562., pid=78, masterSystemTime=1732648479489 2024-11-26T19:14:39,575 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,576 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,576 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,577 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:39,578 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:39,579 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:39,579 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,579 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => f60803f73a60bfffe627ae76fbbfd2d6, NAME => 'test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-26T19:14:39,580 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,580 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:39,580 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,580 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,581 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 604c89e3d00f984e4d556569da533f59; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67352608, jitterRate=0.003632068634033203}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:39,581 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:39,581 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 604c89e3d00f984e4d556569da533f59: Running coprocessor pre-open hook at 1732648479546Writing region info on filesystem at 1732648479546Initializing all the Stores at 1732648479548 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648479548Cleaning up temporary data from old regions at 1732648479576 (+28 ms)Running coprocessor post-open hooks at 1732648479581 (+5 ms)Region opened successfully at 1732648479581 2024-11-26T19:14:39,582 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59., pid=75, masterSystemTime=1732648479506 2024-11-26T19:14:39,583 INFO [StoreOpener-f60803f73a60bfffe627ae76fbbfd2d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,584 INFO [StoreOpener-f60803f73a60bfffe627ae76fbbfd2d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f60803f73a60bfffe627ae76fbbfd2d6 columnFamilyName family 2024-11-26T19:14:39,585 DEBUG [StoreOpener-f60803f73a60bfffe627ae76fbbfd2d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:39,585 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:39,585 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:39,585 INFO [StoreOpener-f60803f73a60bfffe627ae76fbbfd2d6-1 {}] regionserver.HStore(327): Store=f60803f73a60bfffe627ae76fbbfd2d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:39,585 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:39,585 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,586 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => ee0ca46968a82bc066976dac0a0a6d1c, NAME => 'test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-26T19:14:39,586 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,586 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,586 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:39,586 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,586 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,588 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,590 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,590 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,591 INFO [StoreOpener-ee0ca46968a82bc066976dac0a0a6d1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,593 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,594 INFO [StoreOpener-ee0ca46968a82bc066976dac0a0a6d1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee0ca46968a82bc066976dac0a0a6d1c columnFamilyName family 2024-11-26T19:14:39,594 DEBUG [StoreOpener-ee0ca46968a82bc066976dac0a0a6d1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:39,594 INFO [StoreOpener-ee0ca46968a82bc066976dac0a0a6d1c-1 {}] regionserver.HStore(327): Store=ee0ca46968a82bc066976dac0a0a6d1c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:39,594 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,596 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,596 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened f60803f73a60bfffe627ae76fbbfd2d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64751914, jitterRate=-0.03512129187583923}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:39,596 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:39,596 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for f60803f73a60bfffe627ae76fbbfd2d6: Running coprocessor pre-open hook at 1732648479581Writing region info on filesystem at 1732648479581Initializing all the Stores at 1732648479582 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648479582Cleaning up temporary data from old regions at 1732648479590 (+8 ms)Running coprocessor post-open hooks at 1732648479596 (+6 ms)Region opened successfully at 1732648479596 2024-11-26T19:14:39,598 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6., pid=79, masterSystemTime=1732648479489 2024-11-26T19:14:39,599 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,600 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,600 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,601 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:39,601 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:39,601 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. 2024-11-26T19:14:39,601 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 80e1f97febf19796e5899d139b69e706, NAME => 'test,fff,1732648474996.80e1f97febf19796e5899d139b69e706.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-26T19:14:39,601 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,602 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,fff,1732648474996.80e1f97febf19796e5899d139b69e706.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:39,603 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,603 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,605 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,606 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened ee0ca46968a82bc066976dac0a0a6d1c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63636474, jitterRate=-0.05174264311790466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:39,606 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:39,606 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for ee0ca46968a82bc066976dac0a0a6d1c: Running coprocessor pre-open hook at 1732648479586Writing region info on filesystem at 1732648479586Initializing all the Stores at 1732648479590 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648479590Cleaning up temporary data from old regions at 1732648479600 (+10 ms)Running coprocessor post-open hooks at 1732648479606 (+6 ms)Region opened successfully at 1732648479606 2024-11-26T19:14:39,608 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c., pid=76, masterSystemTime=1732648479506 2024-11-26T19:14:39,610 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,610 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,610 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,611 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:39,611 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:39,611 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. 2024-11-26T19:14:39,611 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 4623e7cd0ae68d169e6e332f3459f4b3, NAME => 'test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-26T19:14:39,611 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,611 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,611 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,611 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,612 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T19:14:39,612 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,612 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,612 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,613 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,617 INFO [StoreOpener-4623e7cd0ae68d169e6e332f3459f4b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,618 INFO [StoreOpener-4623e7cd0ae68d169e6e332f3459f4b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4623e7cd0ae68d169e6e332f3459f4b3 columnFamilyName family 2024-11-26T19:14:39,618 DEBUG [StoreOpener-4623e7cd0ae68d169e6e332f3459f4b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:39,619 INFO [StoreOpener-4623e7cd0ae68d169e6e332f3459f4b3-1 {}] regionserver.HStore(327): Store=4623e7cd0ae68d169e6e332f3459f4b3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:39,619 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,620 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,621 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,622 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,622 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,623 INFO [StoreOpener-80e1f97febf19796e5899d139b69e706-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,624 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,625 INFO [StoreOpener-80e1f97febf19796e5899d139b69e706-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80e1f97febf19796e5899d139b69e706 columnFamilyName family 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [StoreOpener-80e1f97febf19796e5899d139b69e706-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,625 INFO [StoreOpener-80e1f97febf19796e5899d139b69e706-1 {}] regionserver.HStore(327): Store=80e1f97febf19796e5899d139b69e706/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T19:14:39,626 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,626 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,626 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 4623e7cd0ae68d169e6e332f3459f4b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60108460, jitterRate=-0.10431414842605591}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:39,626 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4623e7cd0ae68d169e6e332f3459f4b3 2024-11-26T19:14:39,626 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,627 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 4623e7cd0ae68d169e6e332f3459f4b3: Running coprocessor pre-open hook at 1732648479612Writing region info on filesystem at 1732648479612Initializing all the Stores at 1732648479614 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648479614Cleaning up temporary data from old regions at 1732648479622 (+8 ms)Running coprocessor post-open hooks at 1732648479626 (+4 ms)Region opened successfully at 1732648479626 2024-11-26T19:14:39,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,628 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,627 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,629 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,629 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,630 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3., pid=77, masterSystemTime=1732648479506 2024-11-26T19:14:39,632 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,634 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 80e1f97febf19796e5899d139b69e706; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71848804, jitterRate=0.07063060998916626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T19:14:39,634 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:39,633 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,634 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 80e1f97febf19796e5899d139b69e706: Running coprocessor pre-open hook at 1732648479603Writing region info on filesystem at 1732648479603Initializing all the Stores at 1732648479604 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732648479605 (+1 ms)Cleaning up temporary data from old regions at 1732648479629 (+24 ms)Running coprocessor post-open hooks at 1732648479634 (+5 ms)Region opened successfully at 1732648479634 2024-11-26T19:14:39,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,638 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732648474996.80e1f97febf19796e5899d139b69e706., pid=72, masterSystemTime=1732648479489 2024-11-26T19:14:39,639 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,641 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "fe76acf732b2" port: 35951 start_code: 1732648471608 } transition { transition_code: OPENED region_info { region_id: 1732648474996 table_name { namespace: "default" qualifier: "test" } start_key: "fff" end_key: "ggg" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 72 initiating_master_active_time: 1732648471966 }; retry (#0) after 1004ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-26T19:14:39,642 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "fe76acf732b2" port: 37087 start_code: 1732648471827 } transition { transition_code: OPENED region_info { region_id: 1732648474996 table_name { namespace: "default" qualifier: "test" } start_key: "vvv" end_key: "www" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1732648471966 }; retry (#0) after 1006ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-26T19:14:39,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d697b7105100e4e004671d5e4c7c0562, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60803f73a60bfffe627ae76fbbfd2d6, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,647 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,649 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,649 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,649 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d697b7105100e4e004671d5e4c7c0562, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,649 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60803f73a60bfffe627ae76fbbfd2d6, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,649 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 36 msec 2024-11-26T19:14:39,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,657 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,657 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,659 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,659 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,661 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,662 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN to state=OFFLINE, location=null, table=test, region=80e1f97febf19796e5899d139b69e706 to restore RIT 2024-11-26T19:14:39,662 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN to state=OFFLINE, location=null, table=test, region=d697b7105100e4e004671d5e4c7c0562 to restore RIT 2024-11-26T19:14:39,667 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509667 2024-11-26T19:14:39,667 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,667 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,669 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,670 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,670 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d697b7105100e4e004671d5e4c7c0562, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,671 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d697b7105100e4e004671d5e4c7c0562, server=fe76acf732b2,35951,1732648471608 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,702 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,702 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539702 2024-11-26T19:14:39,702 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599702 2024-11-26T19:14:39,702 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 35 msec 2024-11-26T19:14:39,702 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,702 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,702 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,703 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,703 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,703 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,703 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,703 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,704 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,707 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,707 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,709 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d697b7105100e4e004671d5e4c7c0562, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,714 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d697b7105100e4e004671d5e4c7c0562, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,715 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-26T19:14:39,716 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,716 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,716 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN to state=OFFLINE, location=null, table=test, region=80e1f97febf19796e5899d139b69e706 to restore RIT 2024-11-26T19:14:39,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN to state=OFFLINE, location=null, table=test, region=d697b7105100e4e004671d5e4c7c0562 to restore RIT 2024-11-26T19:14:39,722 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509721 2024-11-26T19:14:39,722 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,722 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,723 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,724 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,724 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,724 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,37087,1732648471827 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,734 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,734 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539734 2024-11-26T19:14:39,734 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599734 2024-11-26T19:14:39,734 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-26T19:14:39,734 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,734 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,734 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,734 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,734 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,734 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,734 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,735 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,735 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,739 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,740 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,743 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,743 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,744 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,744 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,744 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,744 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4623e7cd0ae68d169e6e332f3459f4b3, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,744 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-26T19:14:39,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,745 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,745 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN to state=OFFLINE, location=null, table=test, region=80e1f97febf19796e5899d139b69e706 to restore RIT 2024-11-26T19:14:39,748 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN to state=OFFLINE, location=null, table=test, region=d697b7105100e4e004671d5e4c7c0562 to restore RIT 2024-11-26T19:14:39,749 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509748 2024-11-26T19:14:39,749 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,749 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,750 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,755 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,755 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,755 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,760 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,760 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539760 2024-11-26T19:14:39,760 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599760 2024-11-26T19:14:39,760 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-26T19:14:39,760 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,760 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,760 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,760 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,761 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,761 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,761 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,761 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,762 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,765 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,765 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,765 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,765 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,765 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,765 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,765 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,765 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,765 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,770 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,770 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,770 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,770 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,770 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,770 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,770 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,770 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,770 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,771 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,771 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,771 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,771 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,771 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee0ca46968a82bc066976dac0a0a6d1c, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,771 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-26T19:14:39,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,772 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,772 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN to state=OFFLINE, location=null, table=test, region=80e1f97febf19796e5899d139b69e706 to restore RIT 2024-11-26T19:14:39,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN to state=OFFLINE, location=null, table=test, region=d697b7105100e4e004671d5e4c7c0562 to restore RIT 2024-11-26T19:14:39,775 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509775 2024-11-26T19:14:39,775 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,775 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,777 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,781 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,781 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,781 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,786 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,786 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539786 2024-11-26T19:14:39,786 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599786 2024-11-26T19:14:39,786 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-26T19:14:39,786 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,786 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,786 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,787 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,787 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,787 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,788 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,795 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,795 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,795 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,795 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,796 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,796 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,796 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,796 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,796 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,796 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,796 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 604c89e3d00f984e4d556569da533f59, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,796 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-26T19:14:39,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,797 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,797 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN to state=OFFLINE, location=null, table=test, region=80e1f97febf19796e5899d139b69e706 to restore RIT 2024-11-26T19:14:39,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN to state=OFFLINE, location=null, table=test, region=d697b7105100e4e004671d5e4c7c0562 to restore RIT 2024-11-26T19:14:39,800 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509800 2024-11-26T19:14:39,800 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,800 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,802 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,805 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,805 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,805 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,811 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,812 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,812 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,812 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539812 2024-11-26T19:14:39,812 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599812 2024-11-26T19:14:39,812 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-26T19:14:39,812 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,812 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,812 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,812 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,813 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,813 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,813 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,813 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,814 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,817 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,818 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,820 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,823 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,824 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,824 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eefd38e6213218f00748a98ad16d4a8, server=fe76acf732b2,37087,1732648471827 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,824 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-26T19:14:39,825 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,825 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,825 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN to state=OFFLINE, location=null, table=test, region=80e1f97febf19796e5899d139b69e706 to restore RIT 2024-11-26T19:14:39,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN to state=OFFLINE, location=null, table=test, region=d697b7105100e4e004671d5e4c7c0562 to restore RIT 2024-11-26T19:14:39,830 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509830 2024-11-26T19:14:39,830 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,830 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,832 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,834 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,835 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,835 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,843 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,843 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539843 2024-11-26T19:14:39,843 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599843 2024-11-26T19:14:39,843 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-26T19:14:39,843 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,843 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,843 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,843 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,843 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,843 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,844 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,844 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,845 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,848 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,848 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,848 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,849 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,854 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,854 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,854 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,854 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,854 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,855 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,855 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,855 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,855 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,855 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,855 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,855 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,855 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-26T19:14:39,856 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,856 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,857 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN to state=OFFLINE, location=null, table=test, region=80e1f97febf19796e5899d139b69e706 to restore RIT 2024-11-26T19:14:39,860 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN to state=OFFLINE, location=null, table=test, region=d697b7105100e4e004671d5e4c7c0562 to restore RIT 2024-11-26T19:14:39,861 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509861 2024-11-26T19:14:39,861 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,861 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,862 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,863 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,863 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,863 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,868 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,873 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,873 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539873 2024-11-26T19:14:39,873 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599873 2024-11-26T19:14:39,873 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-26T19:14:39,873 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,873 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,873 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,873 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,873 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,873 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,873 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,874 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,874 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,878 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,879 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,883 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80e1f97febf19796e5899d139b69e706, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,885 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-26T19:14:39,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,886 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,886 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,888 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,888 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,888 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN to state=OFFLINE, location=null, table=test, region=80e1f97febf19796e5899d139b69e706 to restore RIT 2024-11-26T19:14:39,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN to state=OFFLINE, location=null, table=test, region=d697b7105100e4e004671d5e4c7c0562 to restore RIT 2024-11-26T19:14:39,889 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509889 2024-11-26T19:14:39,889 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,889 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,890 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,893 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,894 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,900 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,900 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539900 2024-11-26T19:14:39,900 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599900 2024-11-26T19:14:39,900 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-26T19:14:39,900 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,900 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,900 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,900 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,901 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,901 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,901 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,901 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,902 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,906 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,906 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,906 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,906 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,906 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,906 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,907 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,907 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,907 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,907 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,907 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN 2024-11-26T19:14:39,907 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-26T19:14:39,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,908 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,908 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,911 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN to state=OFFLINE, location=null, table=test, region=80e1f97febf19796e5899d139b69e706 to restore RIT 2024-11-26T19:14:39,911 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d697b7105100e4e004671d5e4c7c0562, ASSIGN to state=OFFLINE, location=null, table=test, region=d697b7105100e4e004671d5e4c7c0562 to restore RIT 2024-11-26T19:14:39,911 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509911 2024-11-26T19:14:39,911 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,911 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,916 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,917 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,917 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,917 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,919 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,919 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539919 2024-11-26T19:14:39,919 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599919 2024-11-26T19:14:39,919 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-26T19:14:39,919 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,919 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,919 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,919 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,919 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,919 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,920 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,920 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,920 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,922 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,924 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,925 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,925 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,925 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,925 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,925 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,925 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,925 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,925 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN 2024-11-26T19:14:39,925 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T19:14:39,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,926 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,926 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,928 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,929 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,929 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,929 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,929 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,929 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=80e1f97febf19796e5899d139b69e706, ASSIGN to state=OFFLINE, location=null, table=test, region=80e1f97febf19796e5899d139b69e706 to restore RIT 2024-11-26T19:14:39,930 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509930 2024-11-26T19:14:39,930 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,930 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,931 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,932 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,933 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,941 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,941 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539941 2024-11-26T19:14:39,941 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599941 2024-11-26T19:14:39,942 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-26T19:14:39,942 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,942 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,942 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,942 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,942 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,942 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,942 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,942 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,943 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,946 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,946 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN 2024-11-26T19:14:39,950 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-26T19:14:39,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,951 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,951 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,953 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0eefd38e6213218f00748a98ad16d4a8, ASSIGN to state=OFFLINE, location=null, table=test, region=0eefd38e6213218f00748a98ad16d4a8 to restore RIT 2024-11-26T19:14:39,954 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509954 2024-11-26T19:14:39,954 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,954 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,955 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,956 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,956 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,957 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,962 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,962 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539962 2024-11-26T19:14:39,962 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599962 2024-11-26T19:14:39,962 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T19:14:39,962 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,962 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,962 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,962 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,963 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,963 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,963 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,963 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,964 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,967 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,967 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,967 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,967 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,968 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,968 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,968 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,968 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN 2024-11-26T19:14:39,968 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-26T19:14:39,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,968 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,969 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,970 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T19:14:39,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,971 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,971 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,971 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,971 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,971 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee0ca46968a82bc066976dac0a0a6d1c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee0ca46968a82bc066976dac0a0a6d1c to restore RIT 2024-11-26T19:14:39,972 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509972 2024-11-26T19:14:39,972 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,973 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,974 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,975 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,975 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,975 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,977 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,978 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:39,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:39,981 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:39,981 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648539981 2024-11-26T19:14:39,981 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648599981 2024-11-26T19:14:39,981 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-26T19:14:39,982 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:39,982 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,982 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:39,982 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:39,982 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:39,982 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:39,982 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:39,982 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:39,983 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:39,985 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:39,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:39,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:39,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:39,987 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,988 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,988 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,988 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,988 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:39,988 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN 2024-11-26T19:14:39,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-26T19:14:39,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:39,989 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:39,990 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:39,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:39,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:39,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:39,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:39,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:39,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f90f88ceda5038692cb1f9ef0b051e2a, ASSIGN to state=OFFLINE, location=null, table=test, region=f90f88ceda5038692cb1f9ef0b051e2a to restore RIT 2024-11-26T19:14:39,994 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648509994 2024-11-26T19:14:39,994 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:39,994 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,995 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:39,997 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:39,997 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:39,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:39,997 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:39,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,003 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,003 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540003 2024-11-26T19:14:40,003 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600003 2024-11-26T19:14:40,003 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-26T19:14:40,003 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:40,003 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,003 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,004 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,004 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:40,004 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:40,004 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:40,004 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:40,005 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:40,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:40,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:40,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:40,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:40,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,010 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN 2024-11-26T19:14:40,010 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T19:14:40,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:40,011 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:40,011 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:40,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:40,016 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:40,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f60803f73a60bfffe627ae76fbbfd2d6, ASSIGN to state=OFFLINE, location=null, table=test, region=f60803f73a60bfffe627ae76fbbfd2d6 to restore RIT 2024-11-26T19:14:40,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,020 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648510020 2024-11-26T19:14:40,021 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,022 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:40,023 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:40,025 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:40,025 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:40,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,025 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-26T19:14:40,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,032 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,032 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540032 2024-11-26T19:14:40,032 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600032 2024-11-26T19:14:40,032 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-26T19:14:40,032 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:40,032 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,032 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,032 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,033 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:40,033 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:40,033 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:40,033 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:40,034 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:40,035 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:40,035 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,035 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:40,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:40,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:40,037 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,038 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:40,038 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,038 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN 2024-11-26T19:14:40,038 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T19:14:40,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:40,039 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:40,040 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:40,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:40,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=604c89e3d00f984e4d556569da533f59, ASSIGN to state=OFFLINE, location=null, table=test, region=604c89e3d00f984e4d556569da533f59 to restore RIT 2024-11-26T19:14:40,047 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648510047 2024-11-26T19:14:40,047 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:40,047 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,049 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:40,050 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:40,050 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:40,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,050 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,054 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,054 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540054 2024-11-26T19:14:40,054 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600054 2024-11-26T19:14:40,054 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T19:14:40,054 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:40,054 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,054 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,054 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,055 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:40,055 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:40,055 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:40,055 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:40,056 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:40,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,060 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN 2024-11-26T19:14:40,060 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-26T19:14:40,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:40,060 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:40,060 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:40,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,063 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4623e7cd0ae68d169e6e332f3459f4b3, ASSIGN to state=OFFLINE, location=null, table=test, region=4623e7cd0ae68d169e6e332f3459f4b3 to restore RIT 2024-11-26T19:14:40,064 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648510064 2024-11-26T19:14:40,064 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:40,064 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,065 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:40,066 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:40,066 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,067 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,070 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,070 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540070 2024-11-26T19:14:40,070 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600070 2024-11-26T19:14:40,070 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-26T19:14:40,070 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:40,070 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,070 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,070 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,070 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:40,070 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:40,070 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:40,070 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:40,071 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:40,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:40,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:40,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,072 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:40,073 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,074 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,074 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-26T19:14:40,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:40,075 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:40,075 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:40,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,078 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648510078 2024-11-26T19:14:40,078 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:40,078 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,079 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:40,081 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:40,081 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,081 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,086 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,086 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540086 2024-11-26T19:14:40,086 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600086 2024-11-26T19:14:40,087 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-26T19:14:40,087 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:40,087 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,087 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,087 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,087 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:40,087 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:40,087 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:40,087 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:40,088 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:40,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:40,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:40,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:40,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:40,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:40,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,092 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,092 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.1732648473840, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,092 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-26T19:14:40,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:40,093 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:40,093 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:40,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,096 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,096 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,098 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648510098 2024-11-26T19:14:40,098 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:40,098 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,099 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:40,101 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:40,101 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,101 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,107 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,107 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540107 2024-11-26T19:14:40,107 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600107 2024-11-26T19:14:40,107 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-26T19:14:40,107 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:40,107 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,107 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,107 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,108 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:40,108 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:40,108 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:40,108 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:40,109 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:40,111 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:40,111 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,111 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,111 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,111 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T19:14:40,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:40,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:40,113 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,113 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,113 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,113 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:40,113 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,113 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,113 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,113 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,113 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,114 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T19:14:40,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:40,115 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:40,115 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:40,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,124 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648510124 2024-11-26T19:14:40,124 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:40,124 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,126 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:40,128 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:40,128 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:40,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,128 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,131 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,131 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540131 2024-11-26T19:14:40,131 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600131 2024-11-26T19:14:40,132 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T19:14:40,132 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:40,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,132 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,132 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,132 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:40,132 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:40,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:40,132 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:40,133 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:40,135 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:40,135 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,135 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,135 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T19:14:40,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:40,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:40,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T19:14:40,137 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,137 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,137 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,137 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-26T19:14:40,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:40,138 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:40,138 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:40,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,142 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648510142 2024-11-26T19:14:40,142 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:40,142 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,143 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:40,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:40,145 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,150 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,150 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540150 2024-11-26T19:14:40,150 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600150 2024-11-26T19:14:40,150 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T19:14:40,150 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:40,150 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,150 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,150 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,150 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:40,150 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:40,150 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:40,150 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:40,151 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:40,153 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:40,153 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,153 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T19:14:40,153 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:40,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:40,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,155 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,155 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-26T19:14:40,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:40,156 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:40,156 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:40,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,158 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,158 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,158 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,159 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648510159 2024-11-26T19:14:40,159 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:40,159 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,160 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:40,161 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:40,161 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,161 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,164 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,166 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,166 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540166 2024-11-26T19:14:40,166 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600166 2024-11-26T19:14:40,167 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T19:14:40,167 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:40,167 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,167 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,167 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,167 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:40,167 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:40,167 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:40,167 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:40,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:40,170 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:40,170 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T19:14:40,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:40,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:40,171 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,172 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,172 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,172 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fe76acf732b2%2C39793%2C1732648471742.meta.1732648474374.meta, worker=fe76acf732b2,35951,1732648471608 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T19:14:40,172 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-26T19:14:40,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:40,173 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:40,173 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:40,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,177 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648510177 2024-11-26T19:14:40,177 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:40,177 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,178 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:40,179 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T19:14:40,179 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,179 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,184 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,184 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540184 2024-11-26T19:14:40,184 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600184 2024-11-26T19:14:40,184 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T19:14:40,184 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T19:14:40,184 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,184 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,184 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,184 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T19:14:40,184 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T19:14:40,184 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T19:14:40,185 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T19:14:40,185 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T19:14:40,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T19:14:40,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T19:14:40,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T19:14:40,188 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true 2024-11-26T19:14:40,188 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T19:14:40,188 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-26T19:14:40,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=61 2024-11-26T19:14:40,189 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fe76acf732b2,35951,1732648471608, table=hbase:meta, region=1588230740 2024-11-26T19:14:40,189 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fe76acf732b2,35951,1732648471608, state=OPEN 2024-11-26T19:14:40,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T19:14:40,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T19:14:40,192 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732648510192 2024-11-26T19:14:40,192 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T19:14:40,192 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T19:14:40,193 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T19:14:40,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure f90f88ceda5038692cb1f9ef0b051e2a, server=fe76acf732b2,35951,1732648471608:java.lang.RuntimeException: inject code bug; ServerCrashProcedure fe76acf732b2,39793,1732648471742, splitWal=true, meta=true exec-time=2.6280 sec 2024-11-26T19:14:40,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=02a6e477a3fdc6610cd970f26fa4f748, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e21282686b8b13bb616e2402c265eba0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=604c89e3d00f984e4d556569da533f59, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17b4309496c03b5e59c2108319874f01, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46694b0adbd3389dc584d68e34855bba, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80e1f97febf19796e5899d139b69e706, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95dce235d9eac4994cdb66aed61e67be, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6567f6d117ca6fa8954460c6e3a8a38b, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0eefd38e6213218f00748a98ad16d4a8, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a405982b7a6ac45a1ee2f1e141411d73, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=da37a162f669759fa05d510d0a2a91e0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f90f88ceda5038692cb1f9ef0b051e2a, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59b39ef631c4901278640a3bc267a00c, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa8886d00b7ca4fe8b1f911651334aff, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6994ebee6aa3384b939e1d921bc57d6d, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee0ca46968a82bc066976dac0a0a6d1c, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a80bff91b1c78c52c7778c440d38079, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d697b7105100e4e004671d5e4c7c0562, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8a29b0b37829b046e26e10712540b43b, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a4e0d60b18f7600d128edab0389cb1b5, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=282d35bc91c009e5603b19e30181f7ce, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4623e7cd0ae68d169e6e332f3459f4b3, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=488125472b9f148364cde55a84ce1e55, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=41f746ca5f963e44ed2cf4d43f5b61c0, regionState=OPEN, lastHost=fe76acf732b2,35951,1732648471608, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f60803f73a60bfffe627ae76fbbfd2d6, regionState=OPENING, lastHost=fe76acf732b2,39793,1732648471742, regionLocation=fe76acf732b2,35951,1732648471608, openSeqNum=2 2024-11-26T19:14:40,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=401b0d5d37e0f999b35af1fa8c724a27, regionState=OPEN, lastHost=fe76acf732b2,37087,1732648471827, regionLocation=fe76acf732b2,37087,1732648471827, openSeqNum=2 2024-11-26T19:14:40,200 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T19:14:40,200 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732648540200 2024-11-26T19:14:40,200 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732648600200 2024-11-26T19:14:40,200 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-26T19:14:40,220 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=347 (was 379), OpenFileDescriptor=616 (was 638), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=481 (was 481), ProcessCount=11 (was 11), AvailableMemoryMB=6307 (was 6358) 2024-11-26T19:14:40,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T19:14:40,220 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T19:14:40,220 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:14:40,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:40,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:40,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T19:14:40,221 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T19:14:40,221 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1240002821, stopped=false 2024-11-26T19:14:40,222 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fe76acf732b2,39807,1732648470659 2024-11-26T19:14:40,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T19:14:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T19:14:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T19:14:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:40,224 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,224 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T19:14:40,225 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:14:40,225 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:14:40,225 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T19:14:40,225 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:14:40,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:40,226 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fe76acf732b2,35951,1732648471608' ***** 2024-11-26T19:14:40,226 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T19:14:40,226 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fe76acf732b2,37087,1732648471827' ***** 2024-11-26T19:14:40,226 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T19:14:40,230 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T19:14:40,230 INFO [RS:0;fe76acf732b2:35951 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T19:14:40,230 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T19:14:40,230 INFO [RS:0;fe76acf732b2:35951 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T19:14:40,230 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for a4e0d60b18f7600d128edab0389cb1b5 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for 41f746ca5f963e44ed2cf4d43f5b61c0 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for da37a162f669759fa05d510d0a2a91e0 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for 282d35bc91c009e5603b19e30181f7ce 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for 46694b0adbd3389dc584d68e34855bba 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for 6994ebee6aa3384b939e1d921bc57d6d 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for 4a80bff91b1c78c52c7778c440d38079 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for e21282686b8b13bb616e2402c265eba0 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for d697b7105100e4e004671d5e4c7c0562 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for 6567f6d117ca6fa8954460c6e3a8a38b 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for f60803f73a60bfffe627ae76fbbfd2d6 2024-11-26T19:14:40,231 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a4e0d60b18f7600d128edab0389cb1b5, disabling compactions & flushes 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(959): stopping server fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,231 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T19:14:40,231 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:40,231 INFO [RS:0;fe76acf732b2:35951 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fe76acf732b2:35951. 2024-11-26T19:14:40,231 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T19:14:40,231 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:40,232 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. after waiting 0 ms 2024-11-26T19:14:40,232 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:40,232 DEBUG [RS:0;fe76acf732b2:35951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:14:40,232 DEBUG [RS:0;fe76acf732b2:35951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:40,232 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T19:14:40,232 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T19:14:40,232 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T19:14:40,232 INFO [RS:2;fe76acf732b2:37087 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T19:14:40,232 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T19:14:40,232 INFO [RS:2;fe76acf732b2:37087 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T19:14:40,232 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:40,233 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-26T19:14:40,233 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T19:14:40,233 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T19:14:40,233 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T19:14:40,233 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T19:14:40,233 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T19:14:40,233 DEBUG [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 282d35bc91c009e5603b19e30181f7ce, 41f746ca5f963e44ed2cf4d43f5b61c0, 46694b0adbd3389dc584d68e34855bba, 4a80bff91b1c78c52c7778c440d38079, 6567f6d117ca6fa8954460c6e3a8a38b, 6994ebee6aa3384b939e1d921bc57d6d, 80e1f97febf19796e5899d139b69e706, a4e0d60b18f7600d128edab0389cb1b5, d697b7105100e4e004671d5e4c7c0562, da37a162f669759fa05d510d0a2a91e0, e21282686b8b13bb616e2402c265eba0, f60803f73a60bfffe627ae76fbbfd2d6, f90f88ceda5038692cb1f9ef0b051e2a 2024-11-26T19:14:40,233 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-26T19:14:40,233 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for aa8886d00b7ca4fe8b1f911651334aff 2024-11-26T19:14:40,233 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for 8a29b0b37829b046e26e10712540b43b 2024-11-26T19:14:40,233 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for 401b0d5d37e0f999b35af1fa8c724a27 2024-11-26T19:14:40,233 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for 604c89e3d00f984e4d556569da533f59 2024-11-26T19:14:40,233 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ee0ca46968a82bc066976dac0a0a6d1c, disabling compactions & flushes 2024-11-26T19:14:40,233 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for 17b4309496c03b5e59c2108319874f01 2024-11-26T19:14:40,233 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:40,233 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for 0eefd38e6213218f00748a98ad16d4a8 2024-11-26T19:14:40,233 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:40,233 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for 02a6e477a3fdc6610cd970f26fa4f748 2024-11-26T19:14:40,234 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. after waiting 0 ms 2024-11-26T19:14:40,234 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for 59b39ef631c4901278640a3bc267a00c 2024-11-26T19:14:40,234 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:40,234 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for a405982b7a6ac45a1ee2f1e141411d73 2024-11-26T19:14:40,234 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for 488125472b9f148364cde55a84ce1e55 2024-11-26T19:14:40,234 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(3091): Received CLOSE for 95dce235d9eac4994cdb66aed61e67be 2024-11-26T19:14:40,234 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(959): stopping server fe76acf732b2,37087,1732648471827 2024-11-26T19:14:40,234 INFO [RS:2;fe76acf732b2:37087 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T19:14:40,234 INFO [RS:2;fe76acf732b2:37087 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;fe76acf732b2:37087. 2024-11-26T19:14:40,234 DEBUG [RS:2;fe76acf732b2:37087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T19:14:40,234 DEBUG [RS:2;fe76acf732b2:37087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:40,234 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-26T19:14:40,234 DEBUG [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(1351): Waiting on 02a6e477a3fdc6610cd970f26fa4f748, 0eefd38e6213218f00748a98ad16d4a8, 17b4309496c03b5e59c2108319874f01, 401b0d5d37e0f999b35af1fa8c724a27, 4623e7cd0ae68d169e6e332f3459f4b3, 488125472b9f148364cde55a84ce1e55, 59b39ef631c4901278640a3bc267a00c, 604c89e3d00f984e4d556569da533f59, 8a29b0b37829b046e26e10712540b43b, 95dce235d9eac4994cdb66aed61e67be, a405982b7a6ac45a1ee2f1e141411d73, aa8886d00b7ca4fe8b1f911651334aff, ee0ca46968a82bc066976dac0a0a6d1c 2024-11-26T19:14:40,240 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/a4e0d60b18f7600d128edab0389cb1b5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,242 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/ee0ca46968a82bc066976dac0a0a6d1c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,245 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:40,245 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:40,245 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a4e0d60b18f7600d128edab0389cb1b5: Waiting for close lock at 1732648480231Running coprocessor pre-close hooks at 1732648480231Disabling compacts and flushes for region at 1732648480231Disabling writes for close at 1732648480232 (+1 ms)Writing region close event to WAL at 1732648480232Running coprocessor post-close hooks at 1732648480242 (+10 ms)Closed at 1732648480245 (+3 ms) 2024-11-26T19:14:40,245 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ee0ca46968a82bc066976dac0a0a6d1c: Waiting for close lock at 1732648480233Running coprocessor pre-close hooks at 1732648480233Disabling compacts and flushes for region at 1732648480233Disabling writes for close at 1732648480234 (+1 ms)Writing region close event to WAL at 1732648480234Running coprocessor post-close hooks at 1732648480243 (+9 ms)Closed at 1732648480245 (+2 ms) 2024-11-26T19:14:40,245 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732648474996.ee0ca46968a82bc066976dac0a0a6d1c. 2024-11-26T19:14:40,245 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732648474996.a4e0d60b18f7600d128edab0389cb1b5. 2024-11-26T19:14:40,246 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f90f88ceda5038692cb1f9ef0b051e2a, disabling compactions & flushes 2024-11-26T19:14:40,246 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aa8886d00b7ca4fe8b1f911651334aff, disabling compactions & flushes 2024-11-26T19:14:40,246 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:40,246 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:40,246 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:40,246 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:40,246 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. after waiting 0 ms 2024-11-26T19:14:40,246 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. after waiting 0 ms 2024-11-26T19:14:40,246 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:40,246 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:40,250 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/info/0be429b5f1024f529e4e4d9996f5fef6 is 138, key is test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59./info:regioninfo/1732648479218/Put/seqid=0 2024-11-26T19:14:40,257 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/aa8886d00b7ca4fe8b1f911651334aff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,259 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:40,259 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aa8886d00b7ca4fe8b1f911651334aff: Waiting for close lock at 1732648480246Running coprocessor pre-close hooks at 1732648480246Disabling compacts and flushes for region at 1732648480246Disabling writes for close at 1732648480246Writing region close event to WAL at 1732648480247 (+1 ms)Running coprocessor post-close hooks at 1732648480258 (+11 ms)Closed at 1732648480259 (+1 ms) 2024-11-26T19:14:40,259 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732648474996.aa8886d00b7ca4fe8b1f911651334aff. 2024-11-26T19:14:40,259 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8a29b0b37829b046e26e10712540b43b, disabling compactions & flushes 2024-11-26T19:14:40,259 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:40,259 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:40,259 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. after waiting 0 ms 2024-11-26T19:14:40,259 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:40,261 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f90f88ceda5038692cb1f9ef0b051e2a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,262 INFO [regionserver/fe76acf732b2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T19:14:40,262 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:40,262 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f90f88ceda5038692cb1f9ef0b051e2a: Waiting for close lock at 1732648480246Running coprocessor pre-close hooks at 1732648480246Disabling compacts and flushes for region at 1732648480246Disabling writes for close at 1732648480246Writing region close event to WAL at 1732648480247 (+1 ms)Running coprocessor post-close hooks at 1732648480262 (+15 ms)Closed at 1732648480262 2024-11-26T19:14:40,262 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732648474996.f90f88ceda5038692cb1f9ef0b051e2a. 2024-11-26T19:14:40,262 INFO [regionserver/fe76acf732b2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T19:14:40,262 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 41f746ca5f963e44ed2cf4d43f5b61c0, disabling compactions & flushes 2024-11-26T19:14:40,262 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:40,263 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:40,263 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. after waiting 0 ms 2024-11-26T19:14:40,263 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:40,266 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/8a29b0b37829b046e26e10712540b43b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,267 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:40,267 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8a29b0b37829b046e26e10712540b43b: Waiting for close lock at 1732648480259Running coprocessor pre-close hooks at 1732648480259Disabling compacts and flushes for region at 1732648480259Disabling writes for close at 1732648480259Writing region close event to WAL at 1732648480261 (+2 ms)Running coprocessor post-close hooks at 1732648480267 (+6 ms)Closed at 1732648480267 2024-11-26T19:14:40,267 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732648474996.8a29b0b37829b046e26e10712540b43b. 2024-11-26T19:14:40,267 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 401b0d5d37e0f999b35af1fa8c724a27, disabling compactions & flushes 2024-11-26T19:14:40,267 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:40,267 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:40,267 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. after waiting 0 ms 2024-11-26T19:14:40,267 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:40,273 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/41f746ca5f963e44ed2cf4d43f5b61c0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,274 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:40,274 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 41f746ca5f963e44ed2cf4d43f5b61c0: Waiting for close lock at 1732648480262Running coprocessor pre-close hooks at 1732648480262Disabling compacts and flushes for region at 1732648480262Disabling writes for close at 1732648480263 (+1 ms)Writing region close event to WAL at 1732648480263Running coprocessor post-close hooks at 1732648480274 (+11 ms)Closed at 1732648480274 2024-11-26T19:14:40,274 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732648474996.41f746ca5f963e44ed2cf4d43f5b61c0. 2024-11-26T19:14:40,274 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing da37a162f669759fa05d510d0a2a91e0, disabling compactions & flushes 2024-11-26T19:14:40,274 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:40,274 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:40,274 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. after waiting 0 ms 2024-11-26T19:14:40,274 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:40,277 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/401b0d5d37e0f999b35af1fa8c724a27/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,279 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:40,279 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 401b0d5d37e0f999b35af1fa8c724a27: Waiting for close lock at 1732648480267Running coprocessor pre-close hooks at 1732648480267Disabling compacts and flushes for region at 1732648480267Disabling writes for close at 1732648480267Writing region close event to WAL at 1732648480269 (+2 ms)Running coprocessor post-close hooks at 1732648480278 (+9 ms)Closed at 1732648480279 (+1 ms) 2024-11-26T19:14:40,279 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732648474996.401b0d5d37e0f999b35af1fa8c724a27. 2024-11-26T19:14:40,279 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 604c89e3d00f984e4d556569da533f59, disabling compactions & flushes 2024-11-26T19:14:40,279 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:40,279 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:40,279 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. after waiting 0 ms 2024-11-26T19:14:40,279 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:40,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741869_1045 (size=8195) 2024-11-26T19:14:40,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741869_1045 (size=8195) 2024-11-26T19:14:40,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741869_1045 (size=8195) 2024-11-26T19:14:40,288 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/da37a162f669759fa05d510d0a2a91e0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,289 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:40,289 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for da37a162f669759fa05d510d0a2a91e0: Waiting for close lock at 1732648480274Running coprocessor pre-close hooks at 1732648480274Disabling compacts and flushes for region at 1732648480274Disabling writes for close at 1732648480274Writing region close event to WAL at 1732648480277 (+3 ms)Running coprocessor post-close hooks at 1732648480289 (+12 ms)Closed at 1732648480289 2024-11-26T19:14:40,289 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732648474996.da37a162f669759fa05d510d0a2a91e0. 2024-11-26T19:14:40,289 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 282d35bc91c009e5603b19e30181f7ce, disabling compactions & flushes 2024-11-26T19:14:40,289 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:40,289 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:40,289 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. after waiting 0 ms 2024-11-26T19:14:40,289 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:40,292 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/604c89e3d00f984e4d556569da533f59/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,293 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:40,293 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 604c89e3d00f984e4d556569da533f59: Waiting for close lock at 1732648480279Running coprocessor pre-close hooks at 1732648480279Disabling compacts and flushes for region at 1732648480279Disabling writes for close at 1732648480279Writing region close event to WAL at 1732648480287 (+8 ms)Running coprocessor post-close hooks at 1732648480293 (+6 ms)Closed at 1732648480293 2024-11-26T19:14:40,293 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732648474996.604c89e3d00f984e4d556569da533f59. 2024-11-26T19:14:40,293 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 17b4309496c03b5e59c2108319874f01, disabling compactions & flushes 2024-11-26T19:14:40,293 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:40,293 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:40,293 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. after waiting 0 ms 2024-11-26T19:14:40,293 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:40,294 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/282d35bc91c009e5603b19e30181f7ce/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,295 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:40,295 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 282d35bc91c009e5603b19e30181f7ce: Waiting for close lock at 1732648480289Running coprocessor pre-close hooks at 1732648480289Disabling compacts and flushes for region at 1732648480289Disabling writes for close at 1732648480289Writing region close event to WAL at 1732648480290 (+1 ms)Running coprocessor post-close hooks at 1732648480294 (+4 ms)Closed at 1732648480294 2024-11-26T19:14:40,295 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732648474996.282d35bc91c009e5603b19e30181f7ce. 2024-11-26T19:14:40,295 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46694b0adbd3389dc584d68e34855bba, disabling compactions & flushes 2024-11-26T19:14:40,295 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:40,295 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:40,295 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. after waiting 0 ms 2024-11-26T19:14:40,295 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:40,301 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/17b4309496c03b5e59c2108319874f01/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,303 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:40,303 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 17b4309496c03b5e59c2108319874f01: Waiting for close lock at 1732648480293Running coprocessor pre-close hooks at 1732648480293Disabling compacts and flushes for region at 1732648480293Disabling writes for close at 1732648480293Writing region close event to WAL at 1732648480296 (+3 ms)Running coprocessor post-close hooks at 1732648480302 (+6 ms)Closed at 1732648480303 (+1 ms) 2024-11-26T19:14:40,303 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732648474996.17b4309496c03b5e59c2108319874f01. 2024-11-26T19:14:40,303 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0eefd38e6213218f00748a98ad16d4a8, disabling compactions & flushes 2024-11-26T19:14:40,303 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:40,303 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:40,303 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. after waiting 0 ms 2024-11-26T19:14:40,303 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:40,305 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/46694b0adbd3389dc584d68e34855bba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,306 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:40,306 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46694b0adbd3389dc584d68e34855bba: Waiting for close lock at 1732648480295Running coprocessor pre-close hooks at 1732648480295Disabling compacts and flushes for region at 1732648480295Disabling writes for close at 1732648480295Writing region close event to WAL at 1732648480300 (+5 ms)Running coprocessor post-close hooks at 1732648480306 (+6 ms)Closed at 1732648480306 2024-11-26T19:14:40,306 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732648474996.46694b0adbd3389dc584d68e34855bba. 2024-11-26T19:14:40,306 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6994ebee6aa3384b939e1d921bc57d6d, disabling compactions & flushes 2024-11-26T19:14:40,307 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:40,307 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:40,307 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. after waiting 0 ms 2024-11-26T19:14:40,307 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:40,308 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/0eefd38e6213218f00748a98ad16d4a8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,309 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:40,309 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0eefd38e6213218f00748a98ad16d4a8: Waiting for close lock at 1732648480303Running coprocessor pre-close hooks at 1732648480303Disabling compacts and flushes for region at 1732648480303Disabling writes for close at 1732648480303Writing region close event to WAL at 1732648480304 (+1 ms)Running coprocessor post-close hooks at 1732648480309 (+5 ms)Closed at 1732648480309 2024-11-26T19:14:40,309 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732648474996.0eefd38e6213218f00748a98ad16d4a8. 2024-11-26T19:14:40,309 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 02a6e477a3fdc6610cd970f26fa4f748, disabling compactions & flushes 2024-11-26T19:14:40,309 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:40,309 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:40,309 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. after waiting 0 ms 2024-11-26T19:14:40,309 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:40,312 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/6994ebee6aa3384b939e1d921bc57d6d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,313 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:40,313 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6994ebee6aa3384b939e1d921bc57d6d: Waiting for close lock at 1732648480306Running coprocessor pre-close hooks at 1732648480306Disabling compacts and flushes for region at 1732648480306Disabling writes for close at 1732648480307 (+1 ms)Writing region close event to WAL at 1732648480308 (+1 ms)Running coprocessor post-close hooks at 1732648480313 (+5 ms)Closed at 1732648480313 2024-11-26T19:14:40,313 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732648474996.6994ebee6aa3384b939e1d921bc57d6d. 2024-11-26T19:14:40,313 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4a80bff91b1c78c52c7778c440d38079, disabling compactions & flushes 2024-11-26T19:14:40,313 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:40,313 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:40,313 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. after waiting 0 ms 2024-11-26T19:14:40,313 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:40,315 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/02a6e477a3fdc6610cd970f26fa4f748/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,315 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:40,315 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 02a6e477a3fdc6610cd970f26fa4f748: Waiting for close lock at 1732648480309Running coprocessor pre-close hooks at 1732648480309Disabling compacts and flushes for region at 1732648480309Disabling writes for close at 1732648480309Writing region close event to WAL at 1732648480310 (+1 ms)Running coprocessor post-close hooks at 1732648480315 (+5 ms)Closed at 1732648480315 2024-11-26T19:14:40,316 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732648474996.02a6e477a3fdc6610cd970f26fa4f748. 2024-11-26T19:14:40,316 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 59b39ef631c4901278640a3bc267a00c, disabling compactions & flushes 2024-11-26T19:14:40,316 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:40,316 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:40,316 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. after waiting 0 ms 2024-11-26T19:14:40,316 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:40,318 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/4a80bff91b1c78c52c7778c440d38079/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,319 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:40,320 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4a80bff91b1c78c52c7778c440d38079: Waiting for close lock at 1732648480313Running coprocessor pre-close hooks at 1732648480313Disabling compacts and flushes for region at 1732648480313Disabling writes for close at 1732648480313Writing region close event to WAL at 1732648480314 (+1 ms)Running coprocessor post-close hooks at 1732648480319 (+5 ms)Closed at 1732648480319 2024-11-26T19:14:40,320 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732648474996.4a80bff91b1c78c52c7778c440d38079. 2024-11-26T19:14:40,320 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/59b39ef631c4901278640a3bc267a00c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,320 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e21282686b8b13bb616e2402c265eba0, disabling compactions & flushes 2024-11-26T19:14:40,320 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:40,320 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:40,320 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. after waiting 0 ms 2024-11-26T19:14:40,320 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:40,321 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:40,321 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 59b39ef631c4901278640a3bc267a00c: Waiting for close lock at 1732648480316Running coprocessor pre-close hooks at 1732648480316Disabling compacts and flushes for region at 1732648480316Disabling writes for close at 1732648480316Writing region close event to WAL at 1732648480316Running coprocessor post-close hooks at 1732648480321 (+5 ms)Closed at 1732648480321 2024-11-26T19:14:40,321 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732648474996.59b39ef631c4901278640a3bc267a00c. 2024-11-26T19:14:40,321 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a405982b7a6ac45a1ee2f1e141411d73, disabling compactions & flushes 2024-11-26T19:14:40,321 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:40,321 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:40,321 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. after waiting 0 ms 2024-11-26T19:14:40,321 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:40,324 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/e21282686b8b13bb616e2402c265eba0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,325 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:40,325 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e21282686b8b13bb616e2402c265eba0: Waiting for close lock at 1732648480320Running coprocessor pre-close hooks at 1732648480320Disabling compacts and flushes for region at 1732648480320Disabling writes for close at 1732648480320Writing region close event to WAL at 1732648480321 (+1 ms)Running coprocessor post-close hooks at 1732648480325 (+4 ms)Closed at 1732648480325 2024-11-26T19:14:40,325 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732648474996.e21282686b8b13bb616e2402c265eba0. 2024-11-26T19:14:40,325 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d697b7105100e4e004671d5e4c7c0562, disabling compactions & flushes 2024-11-26T19:14:40,326 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:40,326 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/a405982b7a6ac45a1ee2f1e141411d73/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,326 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:40,326 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. after waiting 0 ms 2024-11-26T19:14:40,326 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:40,326 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:40,326 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a405982b7a6ac45a1ee2f1e141411d73: Waiting for close lock at 1732648480321Running coprocessor pre-close hooks at 1732648480321Disabling compacts and flushes for region at 1732648480321Disabling writes for close at 1732648480321Writing region close event to WAL at 1732648480322 (+1 ms)Running coprocessor post-close hooks at 1732648480326 (+4 ms)Closed at 1732648480326 2024-11-26T19:14:40,326 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732648474996.a405982b7a6ac45a1ee2f1e141411d73. 2024-11-26T19:14:40,327 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 488125472b9f148364cde55a84ce1e55, disabling compactions & flushes 2024-11-26T19:14:40,327 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:40,327 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:40,327 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732648474996.488125472b9f148364cde55a84ce1e55. after waiting 0 ms 2024-11-26T19:14:40,327 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:40,330 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/d697b7105100e4e004671d5e4c7c0562/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,331 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:40,331 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d697b7105100e4e004671d5e4c7c0562: Waiting for close lock at 1732648480325Running coprocessor pre-close hooks at 1732648480325Disabling compacts and flushes for region at 1732648480325Disabling writes for close at 1732648480326 (+1 ms)Writing region close event to WAL at 1732648480326Running coprocessor post-close hooks at 1732648480331 (+5 ms)Closed at 1732648480331 2024-11-26T19:14:40,331 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732648474996.d697b7105100e4e004671d5e4c7c0562. 2024-11-26T19:14:40,331 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6567f6d117ca6fa8954460c6e3a8a38b, disabling compactions & flushes 2024-11-26T19:14:40,331 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:40,331 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:40,331 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. after waiting 0 ms 2024-11-26T19:14:40,331 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:40,331 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/488125472b9f148364cde55a84ce1e55/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,332 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:40,332 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 488125472b9f148364cde55a84ce1e55: Waiting for close lock at 1732648480327Running coprocessor pre-close hooks at 1732648480327Disabling compacts and flushes for region at 1732648480327Disabling writes for close at 1732648480327Writing region close event to WAL at 1732648480327Running coprocessor post-close hooks at 1732648480332 (+5 ms)Closed at 1732648480332 2024-11-26T19:14:40,332 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732648474996.488125472b9f148364cde55a84ce1e55. 2024-11-26T19:14:40,332 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 95dce235d9eac4994cdb66aed61e67be, disabling compactions & flushes 2024-11-26T19:14:40,333 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:40,333 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:40,333 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. after waiting 0 ms 2024-11-26T19:14:40,333 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:40,336 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/6567f6d117ca6fa8954460c6e3a8a38b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,336 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:40,337 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6567f6d117ca6fa8954460c6e3a8a38b: Waiting for close lock at 1732648480331Running coprocessor pre-close hooks at 1732648480331Disabling compacts and flushes for region at 1732648480331Disabling writes for close at 1732648480331Writing region close event to WAL at 1732648480331Running coprocessor post-close hooks at 1732648480336 (+5 ms)Closed at 1732648480336 2024-11-26T19:14:40,337 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732648474996.6567f6d117ca6fa8954460c6e3a8a38b. 2024-11-26T19:14:40,337 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/95dce235d9eac4994cdb66aed61e67be/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,337 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f60803f73a60bfffe627ae76fbbfd2d6, disabling compactions & flushes 2024-11-26T19:14:40,337 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:40,337 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:40,337 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. after waiting 0 ms 2024-11-26T19:14:40,337 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:40,337 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:40,338 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 95dce235d9eac4994cdb66aed61e67be: Waiting for close lock at 1732648480332Running coprocessor pre-close hooks at 1732648480332Disabling compacts and flushes for region at 1732648480332Disabling writes for close at 1732648480333 (+1 ms)Writing region close event to WAL at 1732648480333Running coprocessor post-close hooks at 1732648480337 (+4 ms)Closed at 1732648480337 2024-11-26T19:14:40,338 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732648474996.95dce235d9eac4994cdb66aed61e67be. 2024-11-26T19:14:40,341 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/default/test/f60803f73a60bfffe627ae76fbbfd2d6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T19:14:40,341 INFO [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:40,342 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f60803f73a60bfffe627ae76fbbfd2d6: Waiting for close lock at 1732648480337Running coprocessor pre-close hooks at 1732648480337Disabling compacts and flushes for region at 1732648480337Disabling writes for close at 1732648480337Writing region close event to WAL at 1732648480337Running coprocessor post-close hooks at 1732648480341 (+4 ms)Closed at 1732648480341 2024-11-26T19:14:40,342 DEBUG [RS_CLOSE_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732648474996.f60803f73a60bfffe627ae76fbbfd2d6. 2024-11-26T19:14:40,433 DEBUG [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:40,434 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(976): stopping server fe76acf732b2,37087,1732648471827; all regions closed. 2024-11-26T19:14:40,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741833_1009 (size=6795) 2024-11-26T19:14:40,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741833_1009 (size=6795) 2024-11-26T19:14:40,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741833_1009 (size=6795) 2024-11-26T19:14:40,441 DEBUG [RS:2;fe76acf732b2:37087 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/oldWALs 2024-11-26T19:14:40,442 INFO [RS:2;fe76acf732b2:37087 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fe76acf732b2%2C37087%2C1732648471827:(num 1732648473830) 2024-11-26T19:14:40,442 DEBUG [RS:2;fe76acf732b2:37087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:40,442 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T19:14:40,442 INFO [RS:2;fe76acf732b2:37087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T19:14:40,442 INFO [RS:2;fe76acf732b2:37087 {}] hbase.ChoreService(370): Chore service for: regionserver/fe76acf732b2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T19:14:40,442 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T19:14:40,442 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T19:14:40,442 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T19:14:40,442 INFO [regionserver/fe76acf732b2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T19:14:40,442 INFO [RS:2;fe76acf732b2:37087 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T19:14:40,443 INFO [RS:2;fe76acf732b2:37087 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37087 2024-11-26T19:14:40,443 WARN [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:40,443 WARN [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3., aborting... java.io.IOException: Failed to report opened region to master: test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:40,444 ERROR [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server fe76acf732b2,37087,1732648471827: Failed to open region test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,vvv,1732648474996.4623e7cd0ae68d169e6e332f3459f4b3. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:40,444 ERROR [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-26T19:14:40,444 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-26T19:14:40,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fe76acf732b2,37087,1732648471827 2024-11-26T19:14:40,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T19:14:40,445 INFO [RS:2;fe76acf732b2:37087 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T19:14:40,445 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-26T19:14:40,447 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fe76acf732b2,37087,1732648471827] 2024-11-26T19:14:40,448 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-26T19:14:40,448 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fe76acf732b2,37087,1732648471827 already deleted, retry=false 2024-11-26T19:14:40,448 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-26T19:14:40,449 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fe76acf732b2,37087,1732648471827 expired; onlineServers=1 2024-11-26T19:14:40,454 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 489933472 }, "NonHeapMemoryUsage": { "committed": 162594816, "init": 7667712, "max": -1, "used": 160010448 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "fe76acf732b2", "queueSize": 121, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 1, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 1, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 39739, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 124, "ProcessCallTime_min": 0, "ProcessCallTime_max": 24, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 6, "ProcessCallTime_median": 12, "ProcessCallTime_75th_percentile": 18, "ProcessCallTime_90th_percentile": 21, "ProcessCallTime_95th_percentile": 22, "ProcessCallTime_98th_percentile": 23, "ProcessCallTime_99th_percentile": 23, "ProcessCallTime_99.9th_percentile": 23, "ProcessCallTime_TimeRangeCount_0-1": 124, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 124, "QueueCallTime_min": 0, "QueueCallTime_max": 31, "QueueCallTime_mean": 2, "QueueCallTime_25th_percentile": 7, "QueueCallTime_median": 15, "QueueCallTime_75th_percentile": 23, "QueueCallTime_90th_percentile": 27, "QueueCallTime_95th_percentile": 29, "QueueCallTime_98th_percentile": 30, "QueueCallTime_99th_percentile": 30, "QueueCallTime_99.9th_percentile": 30, "QueueCallTime_TimeRangeCount_0-1": 124, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 124, "TotalCallTime_min": 0, "TotalCallTime_max": 32, "TotalCallTime_mean": 6, "TotalCallTime_25th_percentile": 8, "TotalCallTime_median": 16, "TotalCallTime_75th_percentile": 24, "TotalCallTime_90th_percentile": 28, "TotalCallTime_95th_percentile": 30, "TotalCallTime_98th_percentile": 31, "TotalCallTime_99th_percentile": 31, "TotalCallTime_99.9th_percentile": 31, "TotalCallTime_TimeRangeCount_0-1": 124, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 124, "ResponseSize_min": 0, "ResponseSize_max": 22854, "ResponseSize_mean": 2954, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17140, "ResponseSize_90th_percentile": 20568, "ResponseSize_95th_percentile": 21711, "ResponseSize_98th_percentile": 22396, "ResponseSize_99th_percentile": 22625, "ResponseSize_99.9th_percentile": 22831, "ResponseSize_SizeRangeCount_0-10": 124, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 124, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 289, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 124, "sentBytes": 284467 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "fe76acf732b2", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56139", "tag.serverName": "fe76acf732b2,37087,1732648471827", "tag.clusterId": "4344240a-10ff-4a1f-a2a7-857ca6573a99", "tag.Context": "regionserver", "tag.Hostname": "fe76acf732b2", "regionCount": 12, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732648471827, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 1, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-26T19:14:40,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:14:40,547 INFO [RS:2;fe76acf732b2:37087 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T19:14:40,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37087-0x10155b36f3b0003, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:14:40,547 INFO [RS:2;fe76acf732b2:37087 {}] regionserver.HRegionServer(1031): Exiting; stopping=fe76acf732b2,37087,1732648471827; zookeeper connection closed. 2024-11-26T19:14:40,548 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@446b6865 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@446b6865 2024-11-26T19:14:40,634 DEBUG [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 80e1f97febf19796e5899d139b69e706 2024-11-26T19:14:40,646 INFO [regionserver/fe76acf732b2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T19:14:40,646 INFO [regionserver/fe76acf732b2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T19:14:40,646 WARN [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,fff,1732648474996.80e1f97febf19796e5899d139b69e706., aborting... java.io.IOException: Failed to report opened region to master: test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:40,646 ERROR [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2440): ***** ABORTING region server fe76acf732b2,35951,1732648471608: Failed to open region test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,fff,1732648474996.80e1f97febf19796e5899d139b69e706. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T19:14:40,647 ERROR [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-26T19:14:40,647 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-26T19:14:40,648 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-26T19:14:40,649 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-26T19:14:40,649 DEBUG [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-26T19:14:40,653 INFO [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 492030624 }, "NonHeapMemoryUsage": { "committed": 162922496, "init": 7667712, "max": -1, "used": 160282312 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "fe76acf732b2", "queueSize": 121, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 1, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 1, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 39739, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 124, "ProcessCallTime_min": 0, "ProcessCallTime_max": 24, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 6, "ProcessCallTime_median": 12, "ProcessCallTime_75th_percentile": 18, "ProcessCallTime_90th_percentile": 21, "ProcessCallTime_95th_percentile": 22, "ProcessCallTime_98th_percentile": 23, "ProcessCallTime_99th_percentile": 23, "ProcessCallTime_99.9th_percentile": 23, "ProcessCallTime_TimeRangeCount_0-1": 124, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 124, "QueueCallTime_min": 0, "QueueCallTime_max": 31, "QueueCallTime_mean": 2, "QueueCallTime_25th_percentile": 7, "QueueCallTime_median": 15, "QueueCallTime_75th_percentile": 23, "QueueCallTime_90th_percentile": 27, "QueueCallTime_95th_percentile": 29, "QueueCallTime_98th_percentile": 30, "QueueCallTime_99th_percentile": 30, "QueueCallTime_99.9th_percentile": 30, "QueueCallTime_TimeRangeCount_0-1": 124, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 124, "TotalCallTime_min": 0, "TotalCallTime_max": 32, "TotalCallTime_mean": 6, "TotalCallTime_25th_percentile": 8, "TotalCallTime_median": 16, "TotalCallTime_75th_percentile": 24, "TotalCallTime_90th_percentile": 28, "TotalCallTime_95th_percentile": 30, "TotalCallTime_98th_percentile": 31, "TotalCallTime_99th_percentile": 31, "TotalCallTime_99.9th_percentile": 31, "TotalCallTime_TimeRangeCount_0-1": 124, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 124, "ResponseSize_min": 0, "ResponseSize_max": 22854, "ResponseSize_mean": 2954, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17140, "ResponseSize_90th_percentile": 20568, "ResponseSize_95th_percentile": 21711, "ResponseSize_98th_percentile": 22396, "ResponseSize_99th_percentile": 22625, "ResponseSize_99.9th_percentile": 22831, "ResponseSize_SizeRangeCount_0-10": 124, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 124, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 289, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 124, "sentBytes": 284467 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "fe76acf732b2", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56139", "tag.serverName": "fe76acf732b2,37087,1732648471827", "tag.clusterId": "4344240a-10ff-4a1f-a2a7-857ca6573a99", "tag.Context": "regionserver", "tag.Hostname": "fe76acf732b2", "regionCount": 12, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732648471827, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 1, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-26T19:14:40,654 WARN [RS_OPEN_REGION-regionserver/fe76acf732b2:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fe76acf732b2:39807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fe76acf732b2:39807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-26T19:14:40,681 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/info/0be429b5f1024f529e4e4d9996f5fef6 2024-11-26T19:14:40,689 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/.tmp/info/0be429b5f1024f529e4e4d9996f5fef6 as hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/info/0be429b5f1024f529e4e4d9996f5fef6 2024-11-26T19:14:40,696 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/info/0be429b5f1024f529e4e4d9996f5fef6, entries=24, sequenceid=71, filesize=8.0 K 2024-11-26T19:14:40,697 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 464ms, sequenceid=71, compaction requested=false 2024-11-26T19:14:40,703 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-26T19:14:40,704 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T19:14:40,704 INFO [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T19:14:40,704 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732648480233Running coprocessor pre-close hooks at 1732648480233Disabling compacts and flushes for region at 1732648480233Disabling writes for close at 1732648480233Obtaining lock to block concurrent updates at 1732648480233Preparing flush snapshotting stores in 1588230740 at 1732648480233Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732648480239 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732648480241 (+2 ms)Flushing 1588230740/info: creating writer at 1732648480241Flushing 1588230740/info: appending metadata at 1732648480249 (+8 ms)Flushing 1588230740/info: closing flushed file at 1732648480249Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@227b3459: reopening flushed file at 1732648480689 (+440 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 464ms, sequenceid=71, compaction requested=false at 1732648480697 (+8 ms)Writing region close event to WAL at 1732648480700 (+3 ms)Running coprocessor post-close hooks at 1732648480704 (+4 ms)Closed at 1732648480704 2024-11-26T19:14:40,704 DEBUG [RS_CLOSE_META-regionserver/fe76acf732b2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T19:14:40,834 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(976): stopping server fe76acf732b2,35951,1732648471608; all regions closed. 2024-11-26T19:14:40,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741865_1041 (size=4204) 2024-11-26T19:14:40,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741865_1041 (size=4204) 2024-11-26T19:14:40,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741865_1041 (size=4204) 2024-11-26T19:14:40,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741834_1010 (size=6805) 2024-11-26T19:14:40,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741834_1010 (size=6805) 2024-11-26T19:14:40,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741834_1010 (size=6805) 2024-11-26T19:14:40,842 DEBUG [RS:0;fe76acf732b2:35951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T19:14:40,842 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T19:14:40,842 INFO [RS:0;fe76acf732b2:35951 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T19:14:40,842 INFO [RS:0;fe76acf732b2:35951 {}] hbase.ChoreService(370): Chore service for: regionserver/fe76acf732b2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T19:14:40,842 INFO [RS:0;fe76acf732b2:35951 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T19:14:40,842 INFO [regionserver/fe76acf732b2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T19:14:40,842 INFO [RS:0;fe76acf732b2:35951 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35951 2024-11-26T19:14:40,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fe76acf732b2,35951,1732648471608 2024-11-26T19:14:40,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T19:14:40,845 INFO [RS:0;fe76acf732b2:35951 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T19:14:40,846 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fe76acf732b2,35951,1732648471608] 2024-11-26T19:14:40,848 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fe76acf732b2,35951,1732648471608 already deleted, retry=false 2024-11-26T19:14:40,849 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fe76acf732b2,35951,1732648471608 expired; onlineServers=0 2024-11-26T19:14:40,849 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fe76acf732b2,39807,1732648470659' ***** 2024-11-26T19:14:40,849 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T19:14:40,849 INFO [M:0;fe76acf732b2:39807 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T19:14:40,849 INFO [M:0;fe76acf732b2:39807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T19:14:40,849 DEBUG [M:0;fe76acf732b2:39807 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T19:14:40,849 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T19:14:40,849 DEBUG [M:0;fe76acf732b2:39807 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T19:14:40,849 DEBUG [master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.small.0-1732648473289 {}] cleaner.HFileCleaner(306): Exit Thread[master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.small.0-1732648473289,5,FailOnTimeoutGroup] 2024-11-26T19:14:40,849 DEBUG [master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.large.0-1732648473265 {}] cleaner.HFileCleaner(306): Exit Thread[master/fe76acf732b2:0:becomeActiveMaster-HFileCleaner.large.0-1732648473265,5,FailOnTimeoutGroup] 2024-11-26T19:14:40,850 INFO [M:0;fe76acf732b2:39807 {}] hbase.ChoreService(370): Chore service for: master/fe76acf732b2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T19:14:40,850 INFO [M:0;fe76acf732b2:39807 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T19:14:40,850 DEBUG [M:0;fe76acf732b2:39807 {}] master.HMaster(1795): Stopping service threads 2024-11-26T19:14:40,850 INFO [M:0;fe76acf732b2:39807 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T19:14:40,850 INFO [M:0;fe76acf732b2:39807 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T19:14:40,850 INFO [M:0;fe76acf732b2:39807 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T19:14:40,850 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T19:14:40,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T19:14:40,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T19:14:40,851 DEBUG [M:0;fe76acf732b2:39807 {}] zookeeper.ZKUtil(347): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T19:14:40,851 WARN [M:0;fe76acf732b2:39807 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T19:14:40,852 INFO [M:0;fe76acf732b2:39807 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/.lastflushedseqids 2024-11-26T19:14:40,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741870_1046 (size=1696) 2024-11-26T19:14:40,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741870_1046 (size=1696) 2024-11-26T19:14:40,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741870_1046 (size=1696) 2024-11-26T19:14:40,868 INFO [M:0;fe76acf732b2:39807 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T19:14:40,868 INFO [M:0;fe76acf732b2:39807 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T19:14:40,868 DEBUG [M:0;fe76acf732b2:39807 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T19:14:40,868 INFO [M:0;fe76acf732b2:39807 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:14:40,868 DEBUG [M:0;fe76acf732b2:39807 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:14:40,869 DEBUG [M:0;fe76acf732b2:39807 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T19:14:40,869 DEBUG [M:0;fe76acf732b2:39807 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:14:40,869 INFO [M:0;fe76acf732b2:39807 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.77 KB heapSize=302.30 KB 2024-11-26T19:14:40,887 DEBUG [M:0;fe76acf732b2:39807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3dbbbbd76e4944eaaba2f10f763cf17d is 82, key is hbase:meta,,1/info:regioninfo/1732648478776/Put/seqid=0 2024-11-26T19:14:40,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741871_1047 (size=6071) 2024-11-26T19:14:40,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741871_1047 (size=6071) 2024-11-26T19:14:40,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741871_1047 (size=6071) 2024-11-26T19:14:40,895 INFO [M:0;fe76acf732b2:39807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3dbbbbd76e4944eaaba2f10f763cf17d 2024-11-26T19:14:40,919 DEBUG [M:0;fe76acf732b2:39807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4beec8c2f3ba4e55a218b3ef4f64a1fc is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1732648480194/Put/seqid=0 2024-11-26T19:14:40,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741872_1048 (size=12801) 2024-11-26T19:14:40,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741872_1048 (size=12801) 2024-11-26T19:14:40,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741872_1048 (size=12801) 2024-11-26T19:14:40,931 INFO [M:0;fe76acf732b2:39807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=247.50 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4beec8c2f3ba4e55a218b3ef4f64a1fc 2024-11-26T19:14:40,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:14:40,946 INFO [RS:0;fe76acf732b2:35951 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T19:14:40,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35951-0x10155b36f3b0001, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:14:40,946 INFO [RS:0;fe76acf732b2:35951 {}] regionserver.HRegionServer(1031): Exiting; stopping=fe76acf732b2,35951,1732648471608; zookeeper connection closed. 2024-11-26T19:14:40,946 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2dea23c5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2dea23c5 2024-11-26T19:14:40,947 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T19:14:40,955 DEBUG [M:0;fe76acf732b2:39807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa7332167d1b47bb85ced283a2f6eddd is 69, key is fe76acf732b2,35951,1732648471608/rs:state/1732648473463/Put/seqid=0 2024-11-26T19:14:40,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741873_1049 (size=5442) 2024-11-26T19:14:40,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741873_1049 (size=5442) 2024-11-26T19:14:40,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741873_1049 (size=5442) 2024-11-26T19:14:40,964 INFO [M:0;fe76acf732b2:39807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa7332167d1b47bb85ced283a2f6eddd 2024-11-26T19:14:40,970 INFO [M:0;fe76acf732b2:39807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa7332167d1b47bb85ced283a2f6eddd 2024-11-26T19:14:40,987 DEBUG [M:0;fe76acf732b2:39807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/db11c5a7b4aa4c07a87b9dce853c2e2e is 52, key is load_balancer_on/state:d/1732648477427/Put/seqid=0 2024-11-26T19:14:40,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741874_1050 (size=5058) 2024-11-26T19:14:40,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741874_1050 (size=5058) 2024-11-26T19:14:40,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741874_1050 (size=5058) 2024-11-26T19:14:40,995 INFO [M:0;fe76acf732b2:39807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/db11c5a7b4aa4c07a87b9dce853c2e2e 2024-11-26T19:14:41,002 DEBUG [M:0;fe76acf732b2:39807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3dbbbbd76e4944eaaba2f10f763cf17d as hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3dbbbbd76e4944eaaba2f10f763cf17d 2024-11-26T19:14:41,008 INFO [M:0;fe76acf732b2:39807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3dbbbbd76e4944eaaba2f10f763cf17d, entries=14, sequenceid=571, filesize=5.9 K 2024-11-26T19:14:41,009 DEBUG [M:0;fe76acf732b2:39807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4beec8c2f3ba4e55a218b3ef4f64a1fc as hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4beec8c2f3ba4e55a218b3ef4f64a1fc 2024-11-26T19:14:41,015 INFO [M:0;fe76acf732b2:39807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4beec8c2f3ba4e55a218b3ef4f64a1fc, entries=79, sequenceid=571, filesize=12.5 K 2024-11-26T19:14:41,016 DEBUG [M:0;fe76acf732b2:39807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa7332167d1b47bb85ced283a2f6eddd as hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fa7332167d1b47bb85ced283a2f6eddd 2024-11-26T19:14:41,022 INFO [M:0;fe76acf732b2:39807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa7332167d1b47bb85ced283a2f6eddd 2024-11-26T19:14:41,022 INFO [M:0;fe76acf732b2:39807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fa7332167d1b47bb85ced283a2f6eddd, entries=3, sequenceid=571, filesize=5.3 K 2024-11-26T19:14:41,024 DEBUG [M:0;fe76acf732b2:39807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/db11c5a7b4aa4c07a87b9dce853c2e2e as hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/db11c5a7b4aa4c07a87b9dce853c2e2e 2024-11-26T19:14:41,030 INFO [M:0;fe76acf732b2:39807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45145/user/jenkins/test-data/468e1114-2b07-bc18-6766-f9fc982fb93e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/db11c5a7b4aa4c07a87b9dce853c2e2e, entries=1, sequenceid=571, filesize=4.9 K 2024-11-26T19:14:41,031 INFO [M:0;fe76acf732b2:39807 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.77 KB/254741, heapSize ~302.24 KB/309496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=571, compaction requested=false 2024-11-26T19:14:41,035 INFO [M:0;fe76acf732b2:39807 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T19:14:41,035 DEBUG [M:0;fe76acf732b2:39807 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732648480868Disabling compacts and flushes for region at 1732648480868Disabling writes for close at 1732648480869 (+1 ms)Obtaining lock to block concurrent updates at 1732648480869Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732648480869Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254741, getHeapSize=309496, getOffHeapSize=0, getCellsCount=710 at 1732648480869Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732648480870 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732648480870Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732648480886 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732648480886Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732648480901 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732648480919 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732648480919Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732648480937 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732648480954 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732648480954Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732648480971 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732648480986 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732648480986Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a5ecdff: reopening flushed file at 1732648481001 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e115b18: reopening flushed file at 1732648481008 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fbd814f: reopening flushed file at 1732648481015 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24cdca17: reopening flushed file at 1732648481023 (+8 ms)Finished flush of dataSize ~248.77 KB/254741, heapSize ~302.24 KB/309496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=571, compaction requested=false at 1732648481031 (+8 ms)Writing region close event to WAL at 1732648481034 (+3 ms)Closed at 1732648481034 2024-11-26T19:14:41,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39887 is added to blk_1073741830_1006 (size=292865) 2024-11-26T19:14:41,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37015 is added to blk_1073741830_1006 (size=292865) 2024-11-26T19:14:41,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741830_1006 (size=292865) 2024-11-26T19:14:41,040 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T19:14:41,040 INFO [M:0;fe76acf732b2:39807 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T19:14:41,040 INFO [M:0;fe76acf732b2:39807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39807 2024-11-26T19:14:41,041 INFO [M:0;fe76acf732b2:39807 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T19:14:41,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:14:41,143 INFO [M:0;fe76acf732b2:39807 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T19:14:41,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x10155b36f3b0000, quorum=127.0.0.1:56139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T19:14:41,147 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@46041a9d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T19:14:41,150 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2a39ea25{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T19:14:41,150 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T19:14:41,150 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48a2d1c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T19:14:41,150 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ed43cd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/hadoop.log.dir/,STOPPED} 2024-11-26T19:14:41,152 WARN [BP-189323584-172.17.0.2-1732648466610 heartbeating to localhost/127.0.0.1:45145 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T19:14:41,152 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T19:14:41,152 WARN [BP-189323584-172.17.0.2-1732648466610 heartbeating to localhost/127.0.0.1:45145 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-189323584-172.17.0.2-1732648466610 (Datanode Uuid e521ff40-7d72-4f96-bd9d-fbacdf0eee14) service to localhost/127.0.0.1:45145 2024-11-26T19:14:41,152 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T19:14:41,154 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data5/current/BP-189323584-172.17.0.2-1732648466610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T19:14:41,154 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data6/current/BP-189323584-172.17.0.2-1732648466610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T19:14:41,155 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T19:14:41,157 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@395ff33a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T19:14:41,157 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d24ac65{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T19:14:41,157 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T19:14:41,157 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6331ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T19:14:41,157 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ff36c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/hadoop.log.dir/,STOPPED} 2024-11-26T19:14:41,159 WARN [BP-189323584-172.17.0.2-1732648466610 heartbeating to localhost/127.0.0.1:45145 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T19:14:41,159 WARN [BP-189323584-172.17.0.2-1732648466610 heartbeating to localhost/127.0.0.1:45145 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-189323584-172.17.0.2-1732648466610 (Datanode Uuid e17e6176-ff11-454f-b067-a0bc85f8b859) service to localhost/127.0.0.1:45145 2024-11-26T19:14:41,159 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T19:14:41,159 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T19:14:41,159 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data3/current/BP-189323584-172.17.0.2-1732648466610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T19:14:41,160 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data4/current/BP-189323584-172.17.0.2-1732648466610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T19:14:41,160 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T19:14:41,162 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65e17246{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T19:14:41,162 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@220c8d80{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T19:14:41,162 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T19:14:41,163 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c7135d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T19:14:41,163 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51c39823{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/hadoop.log.dir/,STOPPED} 2024-11-26T19:14:41,164 WARN [BP-189323584-172.17.0.2-1732648466610 heartbeating to localhost/127.0.0.1:45145 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T19:14:41,164 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T19:14:41,164 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T19:14:41,164 WARN [BP-189323584-172.17.0.2-1732648466610 heartbeating to localhost/127.0.0.1:45145 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-189323584-172.17.0.2-1732648466610 (Datanode Uuid 7eb2bab3-e40a-4ec6-814c-8891a0b06d7b) service to localhost/127.0.0.1:45145 2024-11-26T19:14:41,165 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data1/current/BP-189323584-172.17.0.2-1732648466610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T19:14:41,165 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/cluster_8f343790-3886-1648-16f6-3a322fdb005f/data/data2/current/BP-189323584-172.17.0.2-1732648466610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T19:14:41,165 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T19:14:41,166 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T19:14:41,167 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T19:14:41,168 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-26T19:14:41,169 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-26T19:14:41,171 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T19:14:41,171 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T19:14:41,177 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T19:14:41,177 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T19:14:41,177 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T19:14:41,178 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T19:14:41,178 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ce6124f-cb3a-9928-8663-9c6ff20d64ce/hadoop.log.dir/,STOPPED} 2024-11-26T19:14:41,189 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T19:14:41,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down